{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "\n",
    "os.environ['GOOGLE_APPLICATION_CREDENTIALS'] = '/home/husein/t5/prepare/mesolitica-tpu.json'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "\n",
    "os.environ['CUDA_VISIBLE_DEVICES'] = '3'\n",
    "\n",
    "from tensor2tensor.data_generators import problem\n",
    "from tensor2tensor.data_generators import text_problems\n",
    "from tensor2tensor.data_generators import translate\n",
    "from tensor2tensor.utils import registry\n",
    "from tensor2tensor import problems\n",
    "import tensorflow as tf\n",
    "import os\n",
    "import logging\n",
    "\n",
    "logger = logging.getLogger()\n",
    "tf.logging.set_verbosity(tf.logging.DEBUG)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sentencepiece as spm\n",
    "\n",
    "vocab = 'sp10m.cased.t5.model'\n",
    "sp = spm.SentencePieceProcessor()\n",
    "sp.Load(vocab)\n",
    "\n",
    "class Encoder:\n",
    "    def __init__(self, sp):\n",
    "        self.sp = sp\n",
    "        self.vocab_size = sp.GetPieceSize() + 100\n",
    "    \n",
    "    def encode(self, s):\n",
    "        return self.sp.EncodeAsIds(s)\n",
    "    \n",
    "    def decode(self, ids, strip_extraneous=False):\n",
    "        return self.sp.DecodeIds(list(ids))\n",
    "    \n",
    "encoder = Encoder(sp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tqdm import tqdm\n",
    "from glob import glob\n",
    "\n",
    "@registry.register_problem\n",
    "class Seq2Seq(text_problems.Text2TextProblem):\n",
    "\n",
    "    @property\n",
    "    def approx_vocab_size(self):\n",
    "        return 32100\n",
    "    \n",
    "    @property\n",
    "    def is_generate_per_split(self):\n",
    "        return False\n",
    "            \n",
    "    def feature_encoders(self, data_dir):\n",
    "        encoder = Encoder(sp)\n",
    "        return {\n",
    "            \"inputs\": encoder,\n",
    "            \"targets\": encoder\n",
    "        }\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "DATA_DIR = os.path.expanduser('t2t-summarization/data')\n",
    "TMP_DIR = os.path.expanduser('t2t-summarization/tmp')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "PROBLEM = 'seq2_seq'\n",
    "t2t_problem = problems.problem(PROBLEM)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'gs://mesolitica-tpu-general/t2t-summarization-small/model.ckpt-50000'"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import tensorflow as tf\n",
    "import os\n",
    "\n",
    "ckpt_path = tf.train.latest_checkpoint('gs://mesolitica-tpu-general/t2t-summarization-small')\n",
    "ckpt_path"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/tensorflow_gan/python/estimator/tpu_gan_estimator.py:42: The name tf.estimator.tpu.TPUEstimator is deprecated. Please use tf.compat.v1.estimator.tpu.TPUEstimator instead.\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/tensorflow_gan/python/estimator/tpu_gan_estimator.py:42: The name tf.estimator.tpu.TPUEstimator is deprecated. Please use tf.compat.v1.estimator.tpu.TPUEstimator instead.\n",
      "\n"
     ]
    }
   ],
   "source": [
    "from tensor2tensor import models\n",
    "from tensor2tensor import problems\n",
    "from tensor2tensor.layers import common_layers\n",
    "from tensor2tensor.utils import trainer_lib\n",
    "from tensor2tensor.utils import t2t_model\n",
    "from tensor2tensor.utils import registry\n",
    "from tensor2tensor.utils import metrics\n",
    "from tensor2tensor.data_generators import problem\n",
    "from tensor2tensor.data_generators import text_problems\n",
    "from tensor2tensor.data_generators import translate\n",
    "from tensor2tensor.utils import registry"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tensor2tensor.layers import modalities\n",
    "from tensor2tensor.layers import common_layers\n",
    "\n",
    "def top_p_logits(logits, p):\n",
    "    with tf.variable_scope('top_p_logits'):\n",
    "        logits_sort = tf.sort(logits, direction = 'DESCENDING')\n",
    "        probs_sort = tf.nn.softmax(logits_sort)\n",
    "        probs_sums = tf.cumsum(probs_sort, axis = 1, exclusive = True)\n",
    "        logits_masked = tf.where(\n",
    "            probs_sums < p, logits_sort, tf.ones_like(logits_sort) * 1000\n",
    "        )  # [batchsize, vocab]\n",
    "        min_logits = tf.reduce_min(\n",
    "            logits_masked, axis = 1, keepdims = True\n",
    "        )  # [batchsize, 1]\n",
    "        return tf.where(\n",
    "            logits < min_logits,\n",
    "            tf.ones_like(logits, dtype = logits.dtype) * -1e10,\n",
    "            logits,\n",
    "        )\n",
    "\n",
    "\n",
    "def sample(translate_model, features):\n",
    "    logits, losses = translate_model(features)\n",
    "    logits_shape = common_layers.shape_list(logits)\n",
    "    logits_p = logits[:,0,:,0,:] / translate_model.hparams.sampling_temp\n",
    "    logits_p = top_p_logits(logits_p, translate_model.hparams.top_p)\n",
    "    reshaped_logits = tf.reshape(logits_p, [-1, logits_shape[-1]])\n",
    "    choices = tf.multinomial(reshaped_logits, 1)\n",
    "    samples = tf.reshape(choices, logits_shape[:-1])\n",
    "    return samples, logits, losses\n",
    "\n",
    "def nucleus_sampling(translate_model, features, decode_length):\n",
    "    \"\"\"A slow greedy inference method.\n",
    "    Quadratic time in decode_length.\n",
    "    Args:\n",
    "      features: an map of string to `Tensor`\n",
    "      decode_length: an integer.  How many additional timesteps to decode.\n",
    "    Returns:\n",
    "      A dict of decoding results {\n",
    "          \"outputs\": integer `Tensor` of decoded ids of shape\n",
    "              [batch_size, <= decode_length] if beam_size == 1 or\n",
    "              [batch_size, top_beams, <= decode_length]\n",
    "          \"scores\": None\n",
    "          \"logits\": `Tensor` of shape [batch_size, time, 1, 1, vocab_size].\n",
    "          \"losses\": a dictionary: {loss-name (string): floating point `Scalar`}\n",
    "      }\n",
    "    \"\"\"\n",
    "    if not features:\n",
    "        features = {}\n",
    "    inputs_old = None\n",
    "    if 'inputs' in features and len(features['inputs'].shape) < 4:\n",
    "        inputs_old = features['inputs']\n",
    "        features['inputs'] = tf.expand_dims(features['inputs'], 2)\n",
    "    # Save the targets in a var and reassign it after the tf.while loop to avoid\n",
    "    # having targets being in a 'while' frame. This ensures targets when used\n",
    "    # in metric functions stays in the same frame as other vars.\n",
    "    targets_old = features.get('targets', None)\n",
    "\n",
    "    target_modality = translate_model._problem_hparams.modality['targets']\n",
    "\n",
    "    def infer_step(recent_output, recent_logits, unused_loss):\n",
    "        \"\"\"Inference step.\"\"\"\n",
    "        if not tf.executing_eagerly():\n",
    "            if translate_model._target_modality_is_real:\n",
    "                dim = translate_model._problem_hparams.vocab_size['targets']\n",
    "                if dim is not None and hasattr(\n",
    "                    translate_model._hparams, 'vocab_divisor'\n",
    "                ):\n",
    "                    dim += (-dim) % translate_model._hparams.vocab_divisor\n",
    "                recent_output.set_shape([None, None, None, dim])\n",
    "            else:\n",
    "                recent_output.set_shape([None, None, None, 1])\n",
    "        padded = tf.pad(recent_output, [[0, 0], [0, 1], [0, 0], [0, 0]])\n",
    "        features['targets'] = padded\n",
    "        # This is inefficient in that it generates samples at all timesteps,\n",
    "        # not just the last one, except if target_modality is pointwise.\n",
    "        samples, logits, losses = sample(translate_model, features)\n",
    "        # Concatenate the already-generated recent_output with last timestep\n",
    "        # of the newly-generated samples.\n",
    "        top = translate_model._hparams.top.get(\n",
    "            'targets', modalities.get_top(target_modality)\n",
    "        )\n",
    "        if getattr(top, 'pointwise', False):\n",
    "            cur_sample = samples[:, -1, :, :]\n",
    "        else:\n",
    "            cur_sample = samples[\n",
    "                :, common_layers.shape_list(recent_output)[1], :, :\n",
    "            ]\n",
    "        if translate_model._target_modality_is_real:\n",
    "            cur_sample = tf.expand_dims(cur_sample, axis = 1)\n",
    "            samples = tf.concat([recent_output, cur_sample], axis = 1)\n",
    "        else:\n",
    "            cur_sample = tf.to_int64(tf.expand_dims(cur_sample, axis = 1))\n",
    "            samples = tf.concat([recent_output, cur_sample], axis = 1)\n",
    "            if not tf.executing_eagerly():\n",
    "                samples.set_shape([None, None, None, 1])\n",
    "\n",
    "        # Assuming we have one shard for logits.\n",
    "        logits = tf.concat([recent_logits, logits[:, -1:]], 1)\n",
    "        loss = sum([l for l in losses.values() if l is not None])\n",
    "        return samples, logits, loss\n",
    "\n",
    "    # Create an initial output tensor. This will be passed\n",
    "    # to the infer_step, which adds one timestep at every iteration.\n",
    "    if 'partial_targets' in features:\n",
    "        initial_output = tf.to_int64(features['partial_targets'])\n",
    "        while len(initial_output.get_shape().as_list()) < 4:\n",
    "            initial_output = tf.expand_dims(initial_output, 2)\n",
    "        batch_size = common_layers.shape_list(initial_output)[0]\n",
    "    else:\n",
    "        batch_size = common_layers.shape_list(features['inputs'])[0]\n",
    "        if translate_model._target_modality_is_real:\n",
    "            dim = translate_model._problem_hparams.vocab_size['targets']\n",
    "            if dim is not None and hasattr(\n",
    "                translate_model._hparams, 'vocab_divisor'\n",
    "            ):\n",
    "                dim += (-dim) % translate_model._hparams.vocab_divisor\n",
    "            initial_output = tf.zeros(\n",
    "                (batch_size, 0, 1, dim), dtype = tf.float32\n",
    "            )\n",
    "        else:\n",
    "            initial_output = tf.zeros((batch_size, 0, 1, 1), dtype = tf.int64)\n",
    "    # Hack: foldl complains when the output shape is less specified than the\n",
    "    # input shape, so we confuse it about the input shape.\n",
    "    initial_output = tf.slice(\n",
    "        initial_output, [0, 0, 0, 0], common_layers.shape_list(initial_output)\n",
    "    )\n",
    "    target_modality = translate_model._problem_hparams.modality['targets']\n",
    "    if (\n",
    "        target_modality == modalities.ModalityType.CLASS_LABEL\n",
    "        or translate_model._problem_hparams.get('regression_targets')\n",
    "    ):\n",
    "        decode_length = 1\n",
    "    else:\n",
    "        if 'partial_targets' in features:\n",
    "            prefix_length = common_layers.shape_list(\n",
    "                features['partial_targets']\n",
    "            )[1]\n",
    "        else:\n",
    "            prefix_length = common_layers.shape_list(features['inputs'])[1]\n",
    "        decode_length = prefix_length + decode_length\n",
    "\n",
    "    # Initial values of result, logits and loss.\n",
    "    result = initial_output\n",
    "    vocab_size = translate_model._problem_hparams.vocab_size['targets']\n",
    "    if vocab_size is not None and hasattr(\n",
    "        translate_model._hparams, 'vocab_divisor'\n",
    "    ):\n",
    "        vocab_size += (-vocab_size) % translate_model._hparams.vocab_divisor\n",
    "    if translate_model._target_modality_is_real:\n",
    "        logits = tf.zeros((batch_size, 0, 1, vocab_size))\n",
    "        logits_shape_inv = [None, None, None, None]\n",
    "    else:\n",
    "        # tensor of shape [batch_size, time, 1, 1, vocab_size]\n",
    "        logits = tf.zeros((batch_size, 0, 1, 1, vocab_size))\n",
    "        logits_shape_inv = [None, None, None, None, None]\n",
    "    if not tf.executing_eagerly():\n",
    "        logits.set_shape(logits_shape_inv)\n",
    "\n",
    "    loss = 0.0\n",
    "\n",
    "    def while_exit_cond(\n",
    "        result, logits, loss\n",
    "    ):  # pylint: disable=unused-argument\n",
    "        \"\"\"Exit the loop either if reach decode_length or EOS.\"\"\"\n",
    "        length = common_layers.shape_list(result)[1]\n",
    "\n",
    "        not_overflow = length < decode_length\n",
    "\n",
    "        if translate_model._problem_hparams.stop_at_eos:\n",
    "\n",
    "            def fn_not_eos():\n",
    "                return tf.not_equal(  # Check if the last predicted element is a EOS\n",
    "                    tf.squeeze(result[:, -1, :, :]), 1\n",
    "                )\n",
    "\n",
    "            not_eos = tf.cond(\n",
    "                # We only check for early stopping if there is at least 1 element (\n",
    "                # otherwise not_eos will crash).\n",
    "                tf.not_equal(length, 0),\n",
    "                fn_not_eos,\n",
    "                lambda: True,\n",
    "            )\n",
    "\n",
    "            return tf.cond(\n",
    "                tf.equal(batch_size, 1),\n",
    "                # If batch_size == 1, we check EOS for early stopping.\n",
    "                lambda: tf.logical_and(not_overflow, not_eos),\n",
    "                # Else, just wait for max length\n",
    "                lambda: not_overflow,\n",
    "            )\n",
    "        return not_overflow\n",
    "\n",
    "    result, logits, loss = tf.while_loop(\n",
    "        while_exit_cond,\n",
    "        infer_step,\n",
    "        [result, logits, loss],\n",
    "        shape_invariants = [\n",
    "            tf.TensorShape([None, None, None, None]),\n",
    "            tf.TensorShape(logits_shape_inv),\n",
    "            tf.TensorShape([]),\n",
    "        ],\n",
    "        back_prop = False,\n",
    "        parallel_iterations = 1,\n",
    "    )\n",
    "    if inputs_old is not None:  # Restore to not confuse Estimator.\n",
    "        features['inputs'] = inputs_old\n",
    "    # Reassign targets back to the previous value.\n",
    "    if targets_old is not None:\n",
    "        features['targets'] = targets_old\n",
    "    losses = {'training': loss}\n",
    "    if 'partial_targets' in features:\n",
    "        partial_target_length = common_layers.shape_list(\n",
    "            features['partial_targets']\n",
    "        )[1]\n",
    "        result = tf.slice(\n",
    "            result, [0, partial_target_length, 0, 0], [-1, -1, -1, -1]\n",
    "        )\n",
    "    return {\n",
    "        'outputs': result,\n",
    "        'scores': None,\n",
    "        'logits': logits,\n",
    "        'losses': losses,\n",
    "    }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/tensorflow_core/python/util/deprecation.py:507: calling count_nonzero (from tensorflow.python.ops.math_ops) with axis is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "reduction_indices is deprecated, use axis instead\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/tensorflow_core/python/util/deprecation.py:507: calling count_nonzero (from tensorflow.python.ops.math_ops) with axis is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "reduction_indices is deprecated, use axis instead\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Setting T2TModel mode to 'infer'\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Setting T2TModel mode to 'infer'\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Setting hparams.dropout to 0.0\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Setting hparams.dropout to 0.0\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Setting hparams.label_smoothing to 0.0\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Setting hparams.label_smoothing to 0.0\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Setting hparams.layer_prepostprocess_dropout to 0.0\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Setting hparams.layer_prepostprocess_dropout to 0.0\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Setting hparams.symbol_dropout to 0.0\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Setting hparams.symbol_dropout to 0.0\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Setting hparams.attention_dropout to 0.0\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Setting hparams.attention_dropout to 0.0\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Setting hparams.relu_dropout to 0.0\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Setting hparams.relu_dropout to 0.0\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Using variable initializer: uniform_unit_scaling\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Using variable initializer: uniform_unit_scaling\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:Entity <function framework at 0x7f349c4010d0> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: LIVE_VARS_IN\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:Entity <function framework at 0x7f349c4010d0> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: LIVE_VARS_IN\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING: Entity <function framework at 0x7f349c4010d0> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: LIVE_VARS_IN\n",
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/tensor2tensor-1.15.7-py3.6.egg/tensor2tensor/utils/t2t_model.py:2253: to_float (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use `tf.cast` instead.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/tensor2tensor-1.15.7-py3.6.egg/tensor2tensor/utils/t2t_model.py:2253: to_float (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use `tf.cast` instead.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/tensor2tensor-1.15.7-py3.6.egg/tensor2tensor/utils/t2t_model.py:1374: to_int32 (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use `tf.cast` instead.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/tensor2tensor-1.15.7-py3.6.egg/tensor2tensor/utils/t2t_model.py:1374: to_int32 (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use `tf.cast` instead.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Transforming feature 'inputs' with symbol_modality_32128_512.bottom\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Transforming feature 'inputs' with symbol_modality_32128_512.bottom\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/tensorflow_core/python/ops/resource_variable_ops.py:1630: calling BaseResourceVariable.__init__ (from tensorflow.python.ops.resource_variable_ops) with constraint is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "If using Keras pass *_constraint arguments to layers.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/tensorflow_core/python/ops/resource_variable_ops.py:1630: calling BaseResourceVariable.__init__ (from tensorflow.python.ops.resource_variable_ops) with constraint is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "If using Keras pass *_constraint arguments to layers.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Transforming feature 'targets' with symbol_modality_32128_512.targets_bottom\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Transforming feature 'targets' with symbol_modality_32128_512.targets_bottom\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Building model body\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Building model body\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/tensor2tensor-1.15.7-py3.6.egg/tensor2tensor/models/transformer.py:95: calling dropout (from tensorflow.python.ops.nn_ops) with keep_prob is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use `rate` instead of `keep_prob`. Rate should be set to `rate = 1 - keep_prob`.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/tensor2tensor-1.15.7-py3.6.egg/tensor2tensor/models/transformer.py:95: calling dropout (from tensorflow.python.ops.nn_ops) with keep_prob is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use `rate` instead of `keep_prob`. Rate should be set to `rate = 1 - keep_prob`.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:Entity <function attention_bias_to_padding at 0x7f3496bc52f0> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:Entity <function attention_bias_to_padding at 0x7f3496bc52f0> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING: Entity <function attention_bias_to_padding at 0x7f3496bc52f0> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: \n",
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/tensor2tensor-1.15.7-py3.6.egg/tensor2tensor/utils/expert_utils.py:621: where (from tensorflow.python.ops.array_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use tf.where in 2.0, which has the same broadcast rule as np.where\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/tensor2tensor-1.15.7-py3.6.egg/tensor2tensor/utils/expert_utils.py:621: where (from tensorflow.python.ops.array_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use tf.where in 2.0, which has the same broadcast rule as np.where\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:Entity <function layers at 0x7f349c1b4d90> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: If not matching a CFG node, must be a block statement: <gast.gast.ImportFrom object at 0x7f34883b2080>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:Entity <function layers at 0x7f349c1b4d90> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: If not matching a CFG node, must be a block statement: <gast.gast.ImportFrom object at 0x7f34883b2080>\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING: Entity <function layers at 0x7f349c1b4d90> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: If not matching a CFG node, must be a block statement: <gast.gast.ImportFrom object at 0x7f34883b2080>\n",
      "INFO:tensorflow:Transforming body output with symbol_modality_32128_512.top\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Transforming body output with symbol_modality_32128_512.top\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/tensor2tensor-1.15.7-py3.6.egg/tensor2tensor/models/transformer.py:1258: to_int64 (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use `tf.cast` instead.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/tensor2tensor-1.15.7-py3.6.egg/tensor2tensor/models/transformer.py:1258: to_int64 (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use `tf.cast` instead.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Using variable initializer: uniform_unit_scaling\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Using variable initializer: uniform_unit_scaling\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Transforming feature 'inputs' with symbol_modality_32128_512.bottom\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Transforming feature 'inputs' with symbol_modality_32128_512.bottom\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Transforming feature 'targets' with symbol_modality_32128_512.targets_bottom\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Transforming feature 'targets' with symbol_modality_32128_512.targets_bottom\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Building model body\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Building model body\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Transforming body output with symbol_modality_32128_512.top\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Transforming body output with symbol_modality_32128_512.top\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Using variable initializer: uniform_unit_scaling\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Using variable initializer: uniform_unit_scaling\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Transforming feature 'inputs' with symbol_modality_32128_512.bottom\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Transforming feature 'inputs' with symbol_modality_32128_512.bottom\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Transforming feature 'targets' with symbol_modality_32128_512.targets_bottom\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Transforming feature 'targets' with symbol_modality_32128_512.targets_bottom\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Building model body\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Building model body\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Transforming body output with symbol_modality_32128_512.top\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Transforming body output with symbol_modality_32128_512.top\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From <ipython-input-9-dfd530a8b54f>:28: multinomial (from tensorflow.python.ops.random_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use `tf.random.categorical` instead.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From <ipython-input-9-dfd530a8b54f>:28: multinomial (from tensorflow.python.ops.random_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use `tf.random.categorical` instead.\n"
     ]
    }
   ],
   "source": [
    "class Model:\n",
    "    def __init__(self, HPARAMS = \"transformer_base\", DATA_DIR = 't2t/data'):\n",
    "        \n",
    "        self.X = tf.placeholder(tf.int32, [None, None])\n",
    "        self.Y = tf.placeholder(tf.int32, [None, None])\n",
    "        self.top_p = tf.placeholder(tf.float32, None)\n",
    "        \n",
    "        self.X_seq_len = tf.count_nonzero(self.X, 1, dtype=tf.int32)\n",
    "        self.maxlen_decode = tf.reduce_max(self.X_seq_len)\n",
    "        #self.maxlen_decode = tf.placeholder(tf.int32, None)\n",
    "        \n",
    "        x = tf.expand_dims(tf.expand_dims(self.X, -1), -1)\n",
    "        y = tf.expand_dims(tf.expand_dims(self.Y, -1), -1)\n",
    "        \n",
    "        features = {\n",
    "            \"inputs\": x,\n",
    "            \"targets\": y,\n",
    "            \"target_space_id\": tf.constant(1, dtype=tf.int32),\n",
    "        }\n",
    "        self.features = features\n",
    "        \n",
    "        Modes = tf.estimator.ModeKeys\n",
    "        hparams = trainer_lib.create_hparams(HPARAMS, data_dir=DATA_DIR, problem_name=PROBLEM)\n",
    "        hparams.filter_size = 2048\n",
    "        hparams.hidden_size = 512\n",
    "        hparams.num_heads = 8\n",
    "        hparams.num_hidden_layers = 6\n",
    "        hparams.vocab_divisor = 128\n",
    "        hparams.dropout = 0.1\n",
    "        hparams.max_length = 1024\n",
    "        \n",
    "        hparams.label_smoothing = 0.0\n",
    "        hparams.shared_embedding_and_softmax_weights = False\n",
    "        hparams.eval_drop_long_sequences = True\n",
    "        hparams.max_length = 1024\n",
    "        hparams.multiproblem_mixing_schedule = 'pretrain'\n",
    "        \n",
    "        hparams.symbol_modality_num_shards = 1\n",
    "        hparams.attention_dropout_broadcast_dims = '0,1'\n",
    "        hparams.relu_dropout_broadcast_dims = '1'\n",
    "        hparams.layer_prepostprocess_dropout_broadcast_dims = '1'\n",
    "        \n",
    "        translate_model = registry.model('transformer')(hparams, Modes.PREDICT)\n",
    "        self.translate_model = translate_model\n",
    "        logits, _ = translate_model(features)\n",
    "        self.logits = logits\n",
    "        translate_model.hparams.top_p = self.top_p\n",
    "        \n",
    "        with tf.variable_scope(tf.get_variable_scope(), reuse=True):\n",
    "            self.fast_result = translate_model._greedy_infer(features, self.maxlen_decode)[\"outputs\"]\n",
    "            self.beam_result = translate_model._beam_decode_slow(\n",
    "                features, self.maxlen_decode, beam_size=3, \n",
    "                top_beams=1, alpha=0.5)[\"outputs\"]\n",
    "            self.nucleus_result = nucleus_sampling(translate_model, features, self.maxlen_decode)[\"outputs\"]\n",
    "            self.nucleus_result = self.nucleus_result[:,:,0,0]\n",
    "        \n",
    "        self.fast_result = tf.identity(self.fast_result, name = 'greedy')\n",
    "        self.beam_result = tf.identity(self.beam_result, name = 'beam')\n",
    "        self.nucleus_result = tf.identity(self.nucleus_result, name = 'nucleus')\n",
    "        \n",
    "tf.reset_default_graph()\n",
    "sess = tf.InteractiveSession()\n",
    "model = Model()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<tf.Variable 'transformer/symbol_modality_32128_512/input_emb/weights_0:0' shape=(32128, 512) dtype=float32>,\n",
       " <tf.Variable 'transformer/symbol_modality_32128_512/target_emb/weights_0:0' shape=(32128, 512) dtype=float32>,\n",
       " <tf.Variable 'transformer/body/target_space_embedding/kernel:0' shape=(32, 512) dtype=float32>,\n",
       " <tf.Variable 'transformer/body/encoder/layer_0/self_attention/layer_prepostprocess/layer_norm/layer_norm_scale:0' shape=(512,) dtype=float32>,\n",
       " <tf.Variable 'transformer/body/encoder/layer_0/self_attention/layer_prepostprocess/layer_norm/layer_norm_bias:0' shape=(512,) dtype=float32>,\n",
       " <tf.Variable 'transformer/body/encoder/layer_0/self_attention/multihead_attention/q/kernel:0' shape=(512, 512) dtype=float32>,\n",
       " <tf.Variable 'transformer/body/encoder/layer_0/self_attention/multihead_attention/k/kernel:0' shape=(512, 512) dtype=float32>,\n",
       " <tf.Variable 'transformer/body/encoder/layer_0/self_attention/multihead_attention/v/kernel:0' shape=(512, 512) dtype=float32>,\n",
       " <tf.Variable 'transformer/body/encoder/layer_0/self_attention/multihead_attention/output_transform/kernel:0' shape=(512, 512) dtype=float32>,\n",
       " <tf.Variable 'transformer/body/encoder/layer_0/ffn/layer_prepostprocess/layer_norm/layer_norm_scale:0' shape=(512,) dtype=float32>,\n",
       " <tf.Variable 'transformer/body/encoder/layer_0/ffn/layer_prepostprocess/layer_norm/layer_norm_bias:0' shape=(512,) dtype=float32>,\n",
       " <tf.Variable 'transformer/body/encoder/layer_0/ffn/conv1/kernel:0' shape=(512, 2048) dtype=float32>,\n",
       " <tf.Variable 'transformer/body/encoder/layer_0/ffn/conv1/bias:0' shape=(2048,) dtype=float32>,\n",
       " <tf.Variable 'transformer/body/encoder/layer_0/ffn/conv2/kernel:0' shape=(2048, 512) dtype=float32>,\n",
       " <tf.Variable 'transformer/body/encoder/layer_0/ffn/conv2/bias:0' shape=(512,) dtype=float32>,\n",
       " <tf.Variable 'transformer/body/encoder/layer_1/self_attention/layer_prepostprocess/layer_norm/layer_norm_scale:0' shape=(512,) dtype=float32>,\n",
       " <tf.Variable 'transformer/body/encoder/layer_1/self_attention/layer_prepostprocess/layer_norm/layer_norm_bias:0' shape=(512,) dtype=float32>,\n",
       " <tf.Variable 'transformer/body/encoder/layer_1/self_attention/multihead_attention/q/kernel:0' shape=(512, 512) dtype=float32>,\n",
       " <tf.Variable 'transformer/body/encoder/layer_1/self_attention/multihead_attention/k/kernel:0' shape=(512, 512) dtype=float32>,\n",
       " <tf.Variable 'transformer/body/encoder/layer_1/self_attention/multihead_attention/v/kernel:0' shape=(512, 512) dtype=float32>,\n",
       " <tf.Variable 'transformer/body/encoder/layer_1/self_attention/multihead_attention/output_transform/kernel:0' shape=(512, 512) dtype=float32>,\n",
       " <tf.Variable 'transformer/body/encoder/layer_1/ffn/layer_prepostprocess/layer_norm/layer_norm_scale:0' shape=(512,) dtype=float32>,\n",
       " <tf.Variable 'transformer/body/encoder/layer_1/ffn/layer_prepostprocess/layer_norm/layer_norm_bias:0' shape=(512,) dtype=float32>,\n",
       " <tf.Variable 'transformer/body/encoder/layer_1/ffn/conv1/kernel:0' shape=(512, 2048) dtype=float32>,\n",
       " <tf.Variable 'transformer/body/encoder/layer_1/ffn/conv1/bias:0' shape=(2048,) dtype=float32>,\n",
       " <tf.Variable 'transformer/body/encoder/layer_1/ffn/conv2/kernel:0' shape=(2048, 512) dtype=float32>,\n",
       " <tf.Variable 'transformer/body/encoder/layer_1/ffn/conv2/bias:0' shape=(512,) dtype=float32>,\n",
       " <tf.Variable 'transformer/body/encoder/layer_2/self_attention/layer_prepostprocess/layer_norm/layer_norm_scale:0' shape=(512,) dtype=float32>,\n",
       " <tf.Variable 'transformer/body/encoder/layer_2/self_attention/layer_prepostprocess/layer_norm/layer_norm_bias:0' shape=(512,) dtype=float32>,\n",
       " <tf.Variable 'transformer/body/encoder/layer_2/self_attention/multihead_attention/q/kernel:0' shape=(512, 512) dtype=float32>,\n",
       " <tf.Variable 'transformer/body/encoder/layer_2/self_attention/multihead_attention/k/kernel:0' shape=(512, 512) dtype=float32>,\n",
       " <tf.Variable 'transformer/body/encoder/layer_2/self_attention/multihead_attention/v/kernel:0' shape=(512, 512) dtype=float32>,\n",
       " <tf.Variable 'transformer/body/encoder/layer_2/self_attention/multihead_attention/output_transform/kernel:0' shape=(512, 512) dtype=float32>,\n",
       " <tf.Variable 'transformer/body/encoder/layer_2/ffn/layer_prepostprocess/layer_norm/layer_norm_scale:0' shape=(512,) dtype=float32>,\n",
       " <tf.Variable 'transformer/body/encoder/layer_2/ffn/layer_prepostprocess/layer_norm/layer_norm_bias:0' shape=(512,) dtype=float32>,\n",
       " <tf.Variable 'transformer/body/encoder/layer_2/ffn/conv1/kernel:0' shape=(512, 2048) dtype=float32>,\n",
       " <tf.Variable 'transformer/body/encoder/layer_2/ffn/conv1/bias:0' shape=(2048,) dtype=float32>,\n",
       " <tf.Variable 'transformer/body/encoder/layer_2/ffn/conv2/kernel:0' shape=(2048, 512) dtype=float32>,\n",
       " <tf.Variable 'transformer/body/encoder/layer_2/ffn/conv2/bias:0' shape=(512,) dtype=float32>,\n",
       " <tf.Variable 'transformer/body/encoder/layer_3/self_attention/layer_prepostprocess/layer_norm/layer_norm_scale:0' shape=(512,) dtype=float32>,\n",
       " <tf.Variable 'transformer/body/encoder/layer_3/self_attention/layer_prepostprocess/layer_norm/layer_norm_bias:0' shape=(512,) dtype=float32>,\n",
       " <tf.Variable 'transformer/body/encoder/layer_3/self_attention/multihead_attention/q/kernel:0' shape=(512, 512) dtype=float32>,\n",
       " <tf.Variable 'transformer/body/encoder/layer_3/self_attention/multihead_attention/k/kernel:0' shape=(512, 512) dtype=float32>,\n",
       " <tf.Variable 'transformer/body/encoder/layer_3/self_attention/multihead_attention/v/kernel:0' shape=(512, 512) dtype=float32>,\n",
       " <tf.Variable 'transformer/body/encoder/layer_3/self_attention/multihead_attention/output_transform/kernel:0' shape=(512, 512) dtype=float32>,\n",
       " <tf.Variable 'transformer/body/encoder/layer_3/ffn/layer_prepostprocess/layer_norm/layer_norm_scale:0' shape=(512,) dtype=float32>,\n",
       " <tf.Variable 'transformer/body/encoder/layer_3/ffn/layer_prepostprocess/layer_norm/layer_norm_bias:0' shape=(512,) dtype=float32>,\n",
       " <tf.Variable 'transformer/body/encoder/layer_3/ffn/conv1/kernel:0' shape=(512, 2048) dtype=float32>,\n",
       " <tf.Variable 'transformer/body/encoder/layer_3/ffn/conv1/bias:0' shape=(2048,) dtype=float32>,\n",
       " <tf.Variable 'transformer/body/encoder/layer_3/ffn/conv2/kernel:0' shape=(2048, 512) dtype=float32>,\n",
       " <tf.Variable 'transformer/body/encoder/layer_3/ffn/conv2/bias:0' shape=(512,) dtype=float32>,\n",
       " <tf.Variable 'transformer/body/encoder/layer_4/self_attention/layer_prepostprocess/layer_norm/layer_norm_scale:0' shape=(512,) dtype=float32>,\n",
       " <tf.Variable 'transformer/body/encoder/layer_4/self_attention/layer_prepostprocess/layer_norm/layer_norm_bias:0' shape=(512,) dtype=float32>,\n",
       " <tf.Variable 'transformer/body/encoder/layer_4/self_attention/multihead_attention/q/kernel:0' shape=(512, 512) dtype=float32>,\n",
       " <tf.Variable 'transformer/body/encoder/layer_4/self_attention/multihead_attention/k/kernel:0' shape=(512, 512) dtype=float32>,\n",
       " <tf.Variable 'transformer/body/encoder/layer_4/self_attention/multihead_attention/v/kernel:0' shape=(512, 512) dtype=float32>,\n",
       " <tf.Variable 'transformer/body/encoder/layer_4/self_attention/multihead_attention/output_transform/kernel:0' shape=(512, 512) dtype=float32>,\n",
       " <tf.Variable 'transformer/body/encoder/layer_4/ffn/layer_prepostprocess/layer_norm/layer_norm_scale:0' shape=(512,) dtype=float32>,\n",
       " <tf.Variable 'transformer/body/encoder/layer_4/ffn/layer_prepostprocess/layer_norm/layer_norm_bias:0' shape=(512,) dtype=float32>,\n",
       " <tf.Variable 'transformer/body/encoder/layer_4/ffn/conv1/kernel:0' shape=(512, 2048) dtype=float32>,\n",
       " <tf.Variable 'transformer/body/encoder/layer_4/ffn/conv1/bias:0' shape=(2048,) dtype=float32>,\n",
       " <tf.Variable 'transformer/body/encoder/layer_4/ffn/conv2/kernel:0' shape=(2048, 512) dtype=float32>,\n",
       " <tf.Variable 'transformer/body/encoder/layer_4/ffn/conv2/bias:0' shape=(512,) dtype=float32>,\n",
       " <tf.Variable 'transformer/body/encoder/layer_5/self_attention/layer_prepostprocess/layer_norm/layer_norm_scale:0' shape=(512,) dtype=float32>,\n",
       " <tf.Variable 'transformer/body/encoder/layer_5/self_attention/layer_prepostprocess/layer_norm/layer_norm_bias:0' shape=(512,) dtype=float32>,\n",
       " <tf.Variable 'transformer/body/encoder/layer_5/self_attention/multihead_attention/q/kernel:0' shape=(512, 512) dtype=float32>,\n",
       " <tf.Variable 'transformer/body/encoder/layer_5/self_attention/multihead_attention/k/kernel:0' shape=(512, 512) dtype=float32>,\n",
       " <tf.Variable 'transformer/body/encoder/layer_5/self_attention/multihead_attention/v/kernel:0' shape=(512, 512) dtype=float32>,\n",
       " <tf.Variable 'transformer/body/encoder/layer_5/self_attention/multihead_attention/output_transform/kernel:0' shape=(512, 512) dtype=float32>,\n",
       " <tf.Variable 'transformer/body/encoder/layer_5/ffn/layer_prepostprocess/layer_norm/layer_norm_scale:0' shape=(512,) dtype=float32>,\n",
       " <tf.Variable 'transformer/body/encoder/layer_5/ffn/layer_prepostprocess/layer_norm/layer_norm_bias:0' shape=(512,) dtype=float32>,\n",
       " <tf.Variable 'transformer/body/encoder/layer_5/ffn/conv1/kernel:0' shape=(512, 2048) dtype=float32>,\n",
       " <tf.Variable 'transformer/body/encoder/layer_5/ffn/conv1/bias:0' shape=(2048,) dtype=float32>,\n",
       " <tf.Variable 'transformer/body/encoder/layer_5/ffn/conv2/kernel:0' shape=(2048, 512) dtype=float32>,\n",
       " <tf.Variable 'transformer/body/encoder/layer_5/ffn/conv2/bias:0' shape=(512,) dtype=float32>,\n",
       " <tf.Variable 'transformer/body/encoder/layer_prepostprocess/layer_norm/layer_norm_scale:0' shape=(512,) dtype=float32>,\n",
       " <tf.Variable 'transformer/body/encoder/layer_prepostprocess/layer_norm/layer_norm_bias:0' shape=(512,) dtype=float32>,\n",
       " <tf.Variable 'transformer/body/decoder/layer_0/self_attention/layer_prepostprocess/layer_norm/layer_norm_scale:0' shape=(512,) dtype=float32>,\n",
       " <tf.Variable 'transformer/body/decoder/layer_0/self_attention/layer_prepostprocess/layer_norm/layer_norm_bias:0' shape=(512,) dtype=float32>,\n",
       " <tf.Variable 'transformer/body/decoder/layer_0/self_attention/multihead_attention/q/kernel:0' shape=(512, 512) dtype=float32>,\n",
       " <tf.Variable 'transformer/body/decoder/layer_0/self_attention/multihead_attention/k/kernel:0' shape=(512, 512) dtype=float32>,\n",
       " <tf.Variable 'transformer/body/decoder/layer_0/self_attention/multihead_attention/v/kernel:0' shape=(512, 512) dtype=float32>,\n",
       " <tf.Variable 'transformer/body/decoder/layer_0/self_attention/multihead_attention/output_transform/kernel:0' shape=(512, 512) dtype=float32>,\n",
       " <tf.Variable 'transformer/body/decoder/layer_0/encdec_attention/layer_prepostprocess/layer_norm/layer_norm_scale:0' shape=(512,) dtype=float32>,\n",
       " <tf.Variable 'transformer/body/decoder/layer_0/encdec_attention/layer_prepostprocess/layer_norm/layer_norm_bias:0' shape=(512,) dtype=float32>,\n",
       " <tf.Variable 'transformer/body/decoder/layer_0/encdec_attention/multihead_attention/q/kernel:0' shape=(512, 512) dtype=float32>,\n",
       " <tf.Variable 'transformer/body/decoder/layer_0/encdec_attention/multihead_attention/k/kernel:0' shape=(512, 512) dtype=float32>,\n",
       " <tf.Variable 'transformer/body/decoder/layer_0/encdec_attention/multihead_attention/v/kernel:0' shape=(512, 512) dtype=float32>,\n",
       " <tf.Variable 'transformer/body/decoder/layer_0/encdec_attention/multihead_attention/output_transform/kernel:0' shape=(512, 512) dtype=float32>,\n",
       " <tf.Variable 'transformer/body/decoder/layer_0/ffn/layer_prepostprocess/layer_norm/layer_norm_scale:0' shape=(512,) dtype=float32>,\n",
       " <tf.Variable 'transformer/body/decoder/layer_0/ffn/layer_prepostprocess/layer_norm/layer_norm_bias:0' shape=(512,) dtype=float32>,\n",
       " <tf.Variable 'transformer/body/decoder/layer_0/ffn/conv1/kernel:0' shape=(512, 2048) dtype=float32>,\n",
       " <tf.Variable 'transformer/body/decoder/layer_0/ffn/conv1/bias:0' shape=(2048,) dtype=float32>,\n",
       " <tf.Variable 'transformer/body/decoder/layer_0/ffn/conv2/kernel:0' shape=(2048, 512) dtype=float32>,\n",
       " <tf.Variable 'transformer/body/decoder/layer_0/ffn/conv2/bias:0' shape=(512,) dtype=float32>,\n",
       " <tf.Variable 'transformer/body/decoder/layer_1/self_attention/layer_prepostprocess/layer_norm/layer_norm_scale:0' shape=(512,) dtype=float32>,\n",
       " <tf.Variable 'transformer/body/decoder/layer_1/self_attention/layer_prepostprocess/layer_norm/layer_norm_bias:0' shape=(512,) dtype=float32>,\n",
       " <tf.Variable 'transformer/body/decoder/layer_1/self_attention/multihead_attention/q/kernel:0' shape=(512, 512) dtype=float32>,\n",
       " <tf.Variable 'transformer/body/decoder/layer_1/self_attention/multihead_attention/k/kernel:0' shape=(512, 512) dtype=float32>,\n",
       " <tf.Variable 'transformer/body/decoder/layer_1/self_attention/multihead_attention/v/kernel:0' shape=(512, 512) dtype=float32>,\n",
       " <tf.Variable 'transformer/body/decoder/layer_1/self_attention/multihead_attention/output_transform/kernel:0' shape=(512, 512) dtype=float32>,\n",
       " <tf.Variable 'transformer/body/decoder/layer_1/encdec_attention/layer_prepostprocess/layer_norm/layer_norm_scale:0' shape=(512,) dtype=float32>,\n",
       " <tf.Variable 'transformer/body/decoder/layer_1/encdec_attention/layer_prepostprocess/layer_norm/layer_norm_bias:0' shape=(512,) dtype=float32>,\n",
       " <tf.Variable 'transformer/body/decoder/layer_1/encdec_attention/multihead_attention/q/kernel:0' shape=(512, 512) dtype=float32>,\n",
       " <tf.Variable 'transformer/body/decoder/layer_1/encdec_attention/multihead_attention/k/kernel:0' shape=(512, 512) dtype=float32>,\n",
       " <tf.Variable 'transformer/body/decoder/layer_1/encdec_attention/multihead_attention/v/kernel:0' shape=(512, 512) dtype=float32>,\n",
       " <tf.Variable 'transformer/body/decoder/layer_1/encdec_attention/multihead_attention/output_transform/kernel:0' shape=(512, 512) dtype=float32>,\n",
       " <tf.Variable 'transformer/body/decoder/layer_1/ffn/layer_prepostprocess/layer_norm/layer_norm_scale:0' shape=(512,) dtype=float32>,\n",
       " <tf.Variable 'transformer/body/decoder/layer_1/ffn/layer_prepostprocess/layer_norm/layer_norm_bias:0' shape=(512,) dtype=float32>,\n",
       " <tf.Variable 'transformer/body/decoder/layer_1/ffn/conv1/kernel:0' shape=(512, 2048) dtype=float32>,\n",
       " <tf.Variable 'transformer/body/decoder/layer_1/ffn/conv1/bias:0' shape=(2048,) dtype=float32>,\n",
       " <tf.Variable 'transformer/body/decoder/layer_1/ffn/conv2/kernel:0' shape=(2048, 512) dtype=float32>,\n",
       " <tf.Variable 'transformer/body/decoder/layer_1/ffn/conv2/bias:0' shape=(512,) dtype=float32>,\n",
       " <tf.Variable 'transformer/body/decoder/layer_2/self_attention/layer_prepostprocess/layer_norm/layer_norm_scale:0' shape=(512,) dtype=float32>,\n",
       " <tf.Variable 'transformer/body/decoder/layer_2/self_attention/layer_prepostprocess/layer_norm/layer_norm_bias:0' shape=(512,) dtype=float32>,\n",
       " <tf.Variable 'transformer/body/decoder/layer_2/self_attention/multihead_attention/q/kernel:0' shape=(512, 512) dtype=float32>,\n",
       " <tf.Variable 'transformer/body/decoder/layer_2/self_attention/multihead_attention/k/kernel:0' shape=(512, 512) dtype=float32>,\n",
       " <tf.Variable 'transformer/body/decoder/layer_2/self_attention/multihead_attention/v/kernel:0' shape=(512, 512) dtype=float32>,\n",
       " <tf.Variable 'transformer/body/decoder/layer_2/self_attention/multihead_attention/output_transform/kernel:0' shape=(512, 512) dtype=float32>,\n",
       " <tf.Variable 'transformer/body/decoder/layer_2/encdec_attention/layer_prepostprocess/layer_norm/layer_norm_scale:0' shape=(512,) dtype=float32>,\n",
       " <tf.Variable 'transformer/body/decoder/layer_2/encdec_attention/layer_prepostprocess/layer_norm/layer_norm_bias:0' shape=(512,) dtype=float32>,\n",
       " <tf.Variable 'transformer/body/decoder/layer_2/encdec_attention/multihead_attention/q/kernel:0' shape=(512, 512) dtype=float32>,\n",
       " <tf.Variable 'transformer/body/decoder/layer_2/encdec_attention/multihead_attention/k/kernel:0' shape=(512, 512) dtype=float32>,\n",
       " <tf.Variable 'transformer/body/decoder/layer_2/encdec_attention/multihead_attention/v/kernel:0' shape=(512, 512) dtype=float32>,\n",
       " <tf.Variable 'transformer/body/decoder/layer_2/encdec_attention/multihead_attention/output_transform/kernel:0' shape=(512, 512) dtype=float32>,\n",
       " <tf.Variable 'transformer/body/decoder/layer_2/ffn/layer_prepostprocess/layer_norm/layer_norm_scale:0' shape=(512,) dtype=float32>,\n",
       " <tf.Variable 'transformer/body/decoder/layer_2/ffn/layer_prepostprocess/layer_norm/layer_norm_bias:0' shape=(512,) dtype=float32>,\n",
       " <tf.Variable 'transformer/body/decoder/layer_2/ffn/conv1/kernel:0' shape=(512, 2048) dtype=float32>,\n",
       " <tf.Variable 'transformer/body/decoder/layer_2/ffn/conv1/bias:0' shape=(2048,) dtype=float32>,\n",
       " <tf.Variable 'transformer/body/decoder/layer_2/ffn/conv2/kernel:0' shape=(2048, 512) dtype=float32>,\n",
       " <tf.Variable 'transformer/body/decoder/layer_2/ffn/conv2/bias:0' shape=(512,) dtype=float32>,\n",
       " <tf.Variable 'transformer/body/decoder/layer_3/self_attention/layer_prepostprocess/layer_norm/layer_norm_scale:0' shape=(512,) dtype=float32>,\n",
       " <tf.Variable 'transformer/body/decoder/layer_3/self_attention/layer_prepostprocess/layer_norm/layer_norm_bias:0' shape=(512,) dtype=float32>,\n",
       " <tf.Variable 'transformer/body/decoder/layer_3/self_attention/multihead_attention/q/kernel:0' shape=(512, 512) dtype=float32>,\n",
       " <tf.Variable 'transformer/body/decoder/layer_3/self_attention/multihead_attention/k/kernel:0' shape=(512, 512) dtype=float32>,\n",
       " <tf.Variable 'transformer/body/decoder/layer_3/self_attention/multihead_attention/v/kernel:0' shape=(512, 512) dtype=float32>,\n",
       " <tf.Variable 'transformer/body/decoder/layer_3/self_attention/multihead_attention/output_transform/kernel:0' shape=(512, 512) dtype=float32>,\n",
       " <tf.Variable 'transformer/body/decoder/layer_3/encdec_attention/layer_prepostprocess/layer_norm/layer_norm_scale:0' shape=(512,) dtype=float32>,\n",
       " <tf.Variable 'transformer/body/decoder/layer_3/encdec_attention/layer_prepostprocess/layer_norm/layer_norm_bias:0' shape=(512,) dtype=float32>,\n",
       " <tf.Variable 'transformer/body/decoder/layer_3/encdec_attention/multihead_attention/q/kernel:0' shape=(512, 512) dtype=float32>,\n",
       " <tf.Variable 'transformer/body/decoder/layer_3/encdec_attention/multihead_attention/k/kernel:0' shape=(512, 512) dtype=float32>,\n",
       " <tf.Variable 'transformer/body/decoder/layer_3/encdec_attention/multihead_attention/v/kernel:0' shape=(512, 512) dtype=float32>,\n",
       " <tf.Variable 'transformer/body/decoder/layer_3/encdec_attention/multihead_attention/output_transform/kernel:0' shape=(512, 512) dtype=float32>,\n",
       " <tf.Variable 'transformer/body/decoder/layer_3/ffn/layer_prepostprocess/layer_norm/layer_norm_scale:0' shape=(512,) dtype=float32>,\n",
       " <tf.Variable 'transformer/body/decoder/layer_3/ffn/layer_prepostprocess/layer_norm/layer_norm_bias:0' shape=(512,) dtype=float32>,\n",
       " <tf.Variable 'transformer/body/decoder/layer_3/ffn/conv1/kernel:0' shape=(512, 2048) dtype=float32>,\n",
       " <tf.Variable 'transformer/body/decoder/layer_3/ffn/conv1/bias:0' shape=(2048,) dtype=float32>,\n",
       " <tf.Variable 'transformer/body/decoder/layer_3/ffn/conv2/kernel:0' shape=(2048, 512) dtype=float32>,\n",
       " <tf.Variable 'transformer/body/decoder/layer_3/ffn/conv2/bias:0' shape=(512,) dtype=float32>,\n",
       " <tf.Variable 'transformer/body/decoder/layer_4/self_attention/layer_prepostprocess/layer_norm/layer_norm_scale:0' shape=(512,) dtype=float32>,\n",
       " <tf.Variable 'transformer/body/decoder/layer_4/self_attention/layer_prepostprocess/layer_norm/layer_norm_bias:0' shape=(512,) dtype=float32>,\n",
       " <tf.Variable 'transformer/body/decoder/layer_4/self_attention/multihead_attention/q/kernel:0' shape=(512, 512) dtype=float32>,\n",
       " <tf.Variable 'transformer/body/decoder/layer_4/self_attention/multihead_attention/k/kernel:0' shape=(512, 512) dtype=float32>,\n",
       " <tf.Variable 'transformer/body/decoder/layer_4/self_attention/multihead_attention/v/kernel:0' shape=(512, 512) dtype=float32>,\n",
       " <tf.Variable 'transformer/body/decoder/layer_4/self_attention/multihead_attention/output_transform/kernel:0' shape=(512, 512) dtype=float32>,\n",
       " <tf.Variable 'transformer/body/decoder/layer_4/encdec_attention/layer_prepostprocess/layer_norm/layer_norm_scale:0' shape=(512,) dtype=float32>,\n",
       " <tf.Variable 'transformer/body/decoder/layer_4/encdec_attention/layer_prepostprocess/layer_norm/layer_norm_bias:0' shape=(512,) dtype=float32>,\n",
       " <tf.Variable 'transformer/body/decoder/layer_4/encdec_attention/multihead_attention/q/kernel:0' shape=(512, 512) dtype=float32>,\n",
       " <tf.Variable 'transformer/body/decoder/layer_4/encdec_attention/multihead_attention/k/kernel:0' shape=(512, 512) dtype=float32>,\n",
       " <tf.Variable 'transformer/body/decoder/layer_4/encdec_attention/multihead_attention/v/kernel:0' shape=(512, 512) dtype=float32>,\n",
       " <tf.Variable 'transformer/body/decoder/layer_4/encdec_attention/multihead_attention/output_transform/kernel:0' shape=(512, 512) dtype=float32>,\n",
       " <tf.Variable 'transformer/body/decoder/layer_4/ffn/layer_prepostprocess/layer_norm/layer_norm_scale:0' shape=(512,) dtype=float32>,\n",
       " <tf.Variable 'transformer/body/decoder/layer_4/ffn/layer_prepostprocess/layer_norm/layer_norm_bias:0' shape=(512,) dtype=float32>,\n",
       " <tf.Variable 'transformer/body/decoder/layer_4/ffn/conv1/kernel:0' shape=(512, 2048) dtype=float32>,\n",
       " <tf.Variable 'transformer/body/decoder/layer_4/ffn/conv1/bias:0' shape=(2048,) dtype=float32>,\n",
       " <tf.Variable 'transformer/body/decoder/layer_4/ffn/conv2/kernel:0' shape=(2048, 512) dtype=float32>,\n",
       " <tf.Variable 'transformer/body/decoder/layer_4/ffn/conv2/bias:0' shape=(512,) dtype=float32>,\n",
       " <tf.Variable 'transformer/body/decoder/layer_5/self_attention/layer_prepostprocess/layer_norm/layer_norm_scale:0' shape=(512,) dtype=float32>,\n",
       " <tf.Variable 'transformer/body/decoder/layer_5/self_attention/layer_prepostprocess/layer_norm/layer_norm_bias:0' shape=(512,) dtype=float32>,\n",
       " <tf.Variable 'transformer/body/decoder/layer_5/self_attention/multihead_attention/q/kernel:0' shape=(512, 512) dtype=float32>,\n",
       " <tf.Variable 'transformer/body/decoder/layer_5/self_attention/multihead_attention/k/kernel:0' shape=(512, 512) dtype=float32>,\n",
       " <tf.Variable 'transformer/body/decoder/layer_5/self_attention/multihead_attention/v/kernel:0' shape=(512, 512) dtype=float32>,\n",
       " <tf.Variable 'transformer/body/decoder/layer_5/self_attention/multihead_attention/output_transform/kernel:0' shape=(512, 512) dtype=float32>,\n",
       " <tf.Variable 'transformer/body/decoder/layer_5/encdec_attention/layer_prepostprocess/layer_norm/layer_norm_scale:0' shape=(512,) dtype=float32>,\n",
       " <tf.Variable 'transformer/body/decoder/layer_5/encdec_attention/layer_prepostprocess/layer_norm/layer_norm_bias:0' shape=(512,) dtype=float32>,\n",
       " <tf.Variable 'transformer/body/decoder/layer_5/encdec_attention/multihead_attention/q/kernel:0' shape=(512, 512) dtype=float32>,\n",
       " <tf.Variable 'transformer/body/decoder/layer_5/encdec_attention/multihead_attention/k/kernel:0' shape=(512, 512) dtype=float32>,\n",
       " <tf.Variable 'transformer/body/decoder/layer_5/encdec_attention/multihead_attention/v/kernel:0' shape=(512, 512) dtype=float32>,\n",
       " <tf.Variable 'transformer/body/decoder/layer_5/encdec_attention/multihead_attention/output_transform/kernel:0' shape=(512, 512) dtype=float32>,\n",
       " <tf.Variable 'transformer/body/decoder/layer_5/ffn/layer_prepostprocess/layer_norm/layer_norm_scale:0' shape=(512,) dtype=float32>,\n",
       " <tf.Variable 'transformer/body/decoder/layer_5/ffn/layer_prepostprocess/layer_norm/layer_norm_bias:0' shape=(512,) dtype=float32>,\n",
       " <tf.Variable 'transformer/body/decoder/layer_5/ffn/conv1/kernel:0' shape=(512, 2048) dtype=float32>,\n",
       " <tf.Variable 'transformer/body/decoder/layer_5/ffn/conv1/bias:0' shape=(2048,) dtype=float32>,\n",
       " <tf.Variable 'transformer/body/decoder/layer_5/ffn/conv2/kernel:0' shape=(2048, 512) dtype=float32>,\n",
       " <tf.Variable 'transformer/body/decoder/layer_5/ffn/conv2/bias:0' shape=(512,) dtype=float32>,\n",
       " <tf.Variable 'transformer/body/decoder/layer_prepostprocess/layer_norm/layer_norm_scale:0' shape=(512,) dtype=float32>,\n",
       " <tf.Variable 'transformer/body/decoder/layer_prepostprocess/layer_norm/layer_norm_bias:0' shape=(512,) dtype=float32>,\n",
       " <tf.Variable 'transformer/symbol_modality_32128_512/softmax/weights_0:0' shape=(32128, 512) dtype=float32>]"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Restoring parameters from gs://mesolitica-tpu-general/t2t-summarization-small/model.ckpt-50000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Restoring parameters from gs://mesolitica-tpu-general/t2t-summarization-small/model.ckpt-50000\n"
     ]
    }
   ],
   "source": [
    "var_lists = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES)\n",
    "saver = tf.train.Saver(var_list = var_lists)\n",
    "saver.restore(sess, ckpt_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "from unidecode import unidecode\n",
    "\n",
    "def cleaning(string):\n",
    "    return re.sub(r'[ ]+', ' ', unidecode(string.replace('\\n', ' '))).strip()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "from unidecode import unidecode\n",
    "from malaya.text.rules import normalized_chars\n",
    "\n",
    "def filter_news(string):\n",
    "    string = string.lower()\n",
    "    return 'javascript is disabled' in string or 'requires javascript' in string or 'javascript' in string \\\n",
    "    or 'président' in string\n",
    "\n",
    "def make_cleaning(s, c_dict):\n",
    "    s = s.translate(c_dict)\n",
    "    return s\n",
    "\n",
    "def transformer_textcleaning(string):\n",
    "    \"\"\"\n",
    "    use by any transformer model before tokenization\n",
    "    \"\"\"\n",
    "    string = unidecode(string)\n",
    "    string = ' '.join(\n",
    "        [make_cleaning(w, normalized_chars) for w in string.split()]\n",
    "    )\n",
    "    string = re.sub('\\(dot\\)', '.', string)\n",
    "    string = (\n",
    "        re.sub(re.findall(r'\\<a(.*?)\\>', string)[0], '', string)\n",
    "        if (len(re.findall(r'\\<a (.*?)\\>', string)) > 0)\n",
    "        and ('href' in re.findall(r'\\<a (.*?)\\>', string)[0])\n",
    "        else string\n",
    "    )\n",
    "    string = re.sub(\n",
    "        r'\\w+:\\/{2}[\\d\\w-]+(\\.[\\d\\w-]+)*(?:(?:\\/[^\\s/]*))*', ' ', string\n",
    "    )\n",
    "    string = string.replace('\\n', ' ')\n",
    "    string = re.sub(r'[ ]+', ' ', string).strip().split()\n",
    "    string = [w for w in string if w[0] != '@']\n",
    "    return ' '.join(string)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "string = \"\"\"\n",
    "KUALA LUMPUR: Hakim Mahkamah Tinggi, Mohd Nazlan Mohd Ghazali menyifatkan kes penyelewengan dana RM42 juta milik SRC International Sdn Bhd dihadapi Datuk Seri Najib Razak adalah kesalahan salah guna kedudukan, pecah amanah jenayah dan pengubahan wang haram yang paling teruk.\n",
    "\n",
    "Mohd Nazlan yang mensabitkan Najib terhadap kesemua tujuh tuduhan dan memerintahkan bekas Perdana Menteri itu dipenjara 12 tahun, dan didenda RM210 juta, berkata ia bukan sahaja disebabkan oleh alasan bagaimana jenayah itu dilakukan, malah kes berprofil tinggi berkenaan turut membabitkan sejumlah wang yang sangat besar.\n",
    "\n",
    "Melalui alasan penghakiman penuh setebal 801 muka surat itu, Mohd Nazlan, berkata kes terbabit mempunyai elemen yang memberikan kesan ke atas kepentingan awam kerana dana RM42 juta itu adalah milik Kementerian Kewangan (Diperbadankan) (MKD) yang berkemungkinan berasal daripada dana pencen Kumpulan Wang Persaraan (Diperbadankan) (KWAP) berjumlah RM4 bilion.\n",
    "\n",
    "\"Dan yang paling penting ia membabitkan individu yang pada ketika itu berada dalam pada tertinggi dalam kerajaan,\" katanya.\n",
    "\n",
    "Pada 28 Julai lalu, Mohd Nazlan memerintahkan Najib dipenjarakan 10 tahun masing-masing bagi tiga tuduhan pecah amanah wang RM42 juta milik SRC.\n",
    "\n",
    "Hakim turut memerintahkan Najib dipenjara 12 tahun dan denda RM210 juta (jika gagal bayar, lima tahun penjara) bagi tuduhan menyalahgunakan kedudukan.\n",
    "\n",
    "Bagi tuduhan pengubahan wang haram pula, Mohd Nazlan memerintahkan Najib dipenjara 10 tahun bagi setiap tuduhan.\n",
    "\n",
    "Sementara itu, Mohd Nazlan berkata, Najib selaku tertuduh tidak menunjukkan penyesalan, malah mempertahankan pembelaan beliau tidak mengetahui mengenai wang RM42 juta milik SRC itu dalam rayuannya bagi diringankan hukuman.\n",
    "\n",
    "\"Tetapi saya tidak boleh menafikan beliau adalah Perdana Menteri negara ini dan tidak boleh mempersoalkan sumbangannya untuk kebaikan dan kesejahteraan masyarakat dalam pelbagai cara kerana beliau adalah Perdana Menteri selama sembilan tahun.\n",
    "\n",
    "\"Sejarah politik akan terus diperdebatkan sama ada dari segi keseimbangan, beliau melakukan lebih banyak kebaikan daripada keburukan.\n",
    "\n",
    "\"Walau apa pun, ia adalah tidak selari dengan idea sesebuah pentadbiran negara yang bersih daripada rasuah yang tidak boleh bertolak ansur dengan sebarang penyalahgunaan kuasa,\" katanya.\n",
    "\n",
    "Mahkamah Rayuan menetapkan pada 15 Oktober ini bagi pengurusan kes rayuan Najib terhadap sabitan dan hukuman terhadapnya.\n",
    "\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "string2 = \"\"\"\n",
    "Gabungan parti Warisan, Pakatan Harapan, dan Upko hari ini mendedahkan calon-calon masing-masing untuk pilihan raya negeri Sabah, tetapi ketika pengumuman itu berlangsung, perwakilan PKR di dewan itu dilihat ‘gelisah’ seperti ‘tidak senang duduk’.\n",
    "\n",
    "Sekumpulan anggota PKR kemudian dilihat meninggalkan dewan di Pusat Konvensyen Antarabangsa Sabah di Kota Kinabalu selepas berbincang dengan ketua PKR Sabah Christina Liew.\n",
    "\n",
    "Semakan senarai-senarai calon berkenaan mendapati PKR hanya memperolehi separuh daripada jumlah kerusi yang diharapkan.\n",
    "\n",
    "Semalam, PKR Sabah mengumumkan akan bertanding di 14 kerusi tetapi ketika Presiden Warisan Shafie Apdal mengumumkan calon gabungan tersebut hari ini, PKR hanya diberikan tujuh kerusi untuk bertanding.\n",
    "\n",
    "Kerusi yang diberikan adalah Api-Api, Inanam, Tempasuk, Tamparuli, Matunggong, Klias, dan Sook.\n",
    "\n",
    "Klias dan Sook adalah dua kerusi yang diberikan kepada PKR, sementara lima kerusi selebihnya pernah ditandingi oleh PKR pada pilihan raya umum 2018.\n",
    "\n",
    "Dalam pengumuman PKR Sabah semalam, parti itu menjangkakan Warisan akan turut menyerahkan kerusi Kemabong, Membakut, dan Petagas kepada mereka.\n",
    "\n",
    "Walau bagaimanapun, Warisan menyerahkan kerusi Kemabong kepada Upko dan mengekalkan bertanding untuk kerusi Membakut dan Petagas.\n",
    "\n",
    "PKR juga menuntut empat daripada 13 kerusi baru yang diperkenalkan iaitu Segama, Limbahau, Sungai Manila, dan Pintasan tetapi Warisan membolot semua kerusi itu.\n",
    "\n",
    "Sebagai pertukaran untuk kerusi yang diintainya, PKR bersedia untuk menyerahkan kerusi Kadaimaian, Kuala Penyu, dan Karanaan. Namun, ini dijangka tidak akan berlaku memandangkan parti tersebut tidak berpuas hati dengan agihan kerusi seperti yang diharapkan itu.\n",
    "\n",
    "Selepas perwakilan dari PKR dan Liew keluar dari dewan tersebut, wartawan kemudian menyusuri Liew untuk mendapatkan penjelasannya.\n",
    "\n",
    "Walau bagaimanapun, Liew enggan memberikan sebarang komen dan berkata bahawa dia ingin ke tandas.\n",
    "\n",
    "Liew dan perwakilan PKR kemudian tidak kembali ke dalam dewan tersebut.\n",
    "\n",
    "Apabila calon pilihan raya yang diumumkan diminta naik ke atas pentas untuk sesi bergambar, Liew tidak kelihatan.\n",
    "\n",
    "Bilangan kerusi yang ditandingi oleh PKR kali ini hanya kurang satu kerusi daripada yang ditandingi parti itu pada PRU 2018.\n",
    "\n",
    "Dalam perkembangan berkaitan, DAP dan Amanah dikatakan tidak mempunyai sebarang masalah dengan kerusi yang diberikan untuk PRN Sabah.\n",
    "\n",
    "Sementara itu, Presiden Upko Madius Tangau enggan mengulas adakah dia berpuas hati dengan agihan kerusi tersebut. Madius kekal di majlis tersebut sehingga ia berakhir.\n",
    "\n",
    "Partinya diberikan 12 kerusi, iaitu lebih tujuh kerusi berbanding PRU lalu.\n",
    "\n",
    "DAP dan Amanah akan bertanding di bawah logo Warisan sementara PKR dan Upko akan menggunakan logo masing-masing.\n",
    "\n",
    "DAP akan bertanding di tujuh kerusi, jumlah yang sama seperti yang mereka tandingi pada PRU lalu, sementara Amanah diberi satu kerusi.\n",
    "\n",
    "Warisan akan bertanding sebanyak 54 kerusi.\n",
    "\n",
    "Perkembangan terbaru ini mungkin mencetuskan pergeseran di antara PKR dan Warisan. PKR boleh memilih untuk bertanding di lebih banyak kerusi daripada 14 yang dituntutnya manakala Warisan juga boleh bertanding di kerusi sekutunya.\n",
    "\n",
    "Barisan pemimpin tertinggi PKR dan Warisan hanya mempunyai dua hari sebelum hari penamaan calon pada Sabtu untuk mengurangkan pergeseran.\n",
    "\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "string3 = \"\"\"\n",
    "Penubuhan universiti sukan seperti diutarakan Ketua Unit Sukan Kementerian Pengajian Tinggi, Dr Pekan Ramli dan disokong Pakar Pembangunan Sukan dan Reakreasi Luar, Universiti Pendidikan Sultan Idris (UPSI), Prof Dr Md Amin Md Taaf seperti disiarkan akhbar ini, memberikan sinar harapan kepada kewujudan institusi sedemikian.\n",
    "\n",
    "Ia menjadi impian atlet negara untuk mengejar kejayaan dalam bidang sukan dan kecemerlangan dalam akademik untuk menjamin masa depan lebih baik apabila bersara daripada arena sukan kelak.\n",
    "\n",
    "Pelbagai pandangan, idea, kaedah, bukti dan cadangan dilontarkan pakar berikutan pentingnya universiti sukan yang akan memberi impak besar sama ada pada peringkat kebangsaan mahupun antarabangsa.\n",
    "\n",
    "Negara lain sudah lama meraih laba dengan kewujudan universiti sukan seperti China, Korea, Japan, Taiwan, India dan Vietnam. Mereka menghasilkan atlet universiti yang mempamerkan keputusan cemerlang pada peringkat tinggi seperti Sukan Olimpik, Kejohanan Dunia dan Sukan Asia.\n",
    "\n",
    "Justeru, kejayaan mereka perlu dijadikan rujukan demi memajukan sukan tanah air. Jika kita merujuk pendekatan Asia, kewujudan universiti sukan penting dan memberi kesan positif dalam melonjakkan prestasi sukan lebih optimum.\n",
    "\n",
    "Namun, jika kita melihat pendekatan Eropah, universiti sukan bukan antara organisasi atau institusi penting yang diberi perhatian dalam menyumbang kepada pemenang pingat.\n",
    "\n",
    "Antara isu dalam universiti sukan ialah kos tinggi, lokasi, prasarana sukan, pertindihan kursus dengan universiti sedia ada dan impak terhadap dunia sukan negara hingga mengundang persoalan kewajaran dan kerelevanan penubuhannya.\n",
    "\n",
    "Namun sebagai bekas atlet memanah negara dan Olympian (OLY) di Sukan Olimpik 2004 di Athens, Greece serta bekas pelajar Sekolah Sukan Bukit Jalil hingga berjaya dalam dunia akademik, saya mendapati terdapat beberapa faktor sering menjadi halangan dalam rutin harian mereka.\n",
    "\n",
    "Antaranya, faktor masa yang terpaksa bergegas menghadiri kuliah selepas tamat sesi latihan yang mengambil masa 15 hingga 20 minit dengan menunggang motosikal; kereta (20-30 minit) atau pengangkutan disediakan Majlis Sukan Negara (MSN) ke Universiti Putra Malaysia (UPM).\n",
    "\n",
    "Jika mereka menuntut di Universiti Teknologi MARA (UiTM) atau Universiti Malaya (UM), ia mungkin lebih lama.\n",
    "\n",
    "Walaupun di universiti tersedia dengan kemudahan kolej dan kemudahan sukan, mereka memilih pulang ke MSN untuk menjalani latihan bersama pasukan dan jurulatih di padang atau gelanggang latihan rasmi.\n",
    "\n",
    "Ini berlanjutan selagi bergelar atlet negara yang perlu memastikan prestasi sentiasa meningkat dari semasa ke semasa tanpa mengabaikan tugas sebagai pelajar.\n",
    "\n",
    "Alangkah baiknya jika sebahagian Sekolah Sukan Bukit Jalil itu sendiri dijadikan Kolej atau Universiti Sukan Malaysia kerana lengkap dari segi kemudahan prasarana sukannya dan proses pengajaran dan pembelajaran (PdP) dalam bidang Sains Sukan, Kejurulatihan, Pendidikan Jasmani dan setaraf dengannya.\n",
    "\n",
    "Pengambilan setiap semester pula hanya terhad kepada atlet berstatus kebangsaan dan antarabangsa sahaja supaya hasrat melahirkan lebih ramai atlet bertaraf Olimpik mudah direalisasikan.\n",
    "\n",
    "Contohnya, bekas atlet lompat bergalah negara, Roslinda Samsu yang juga pemenang pingat perak Sukan Asia Doha 2006 dan Penerima Anugerah Khas Majlis Anugerah Sukan KPT 2012, terpaksa mengambil masa lebih kurang sembilan tahun untuk menamatkan ijazah Sarjana Muda Pendidikan Jasmani di UPM sepanjang 14 tahun terbabit dalam sukan olahraga.\n",
    "\n",
    "Sepanjang tempoh bergelar atlet kebangsaan dan mahasiswa, beliau juga memenangi pingat Emas Sukan SEA empat siri berturut-turut pada 2005, 2007, 2009 dan 2011.\n",
    "\n",
    "Begitu juga atlet kebangsaan seperti Leong Mun Yee (UPM); Pandalela Renong (UM); Bryan Nickson Lomas (UM); Cheng Chu Sian (UPM); Marbawi Sulaiman (UiTM) dan Norasheela Khalid (UPM).\n",
    "\n",
    "Jika disenaraikan, mungkin lebih ramai lagi. Namun, pernah terlintas di fikiran mengapa hanya atlet dari sukan terjun yang dapat memenangi pingat di Sukan Olimpik? Bagaimana dengan atlet lain yang juga layak secara merit? Apakah kekangan atau masalah dihadapi sebagai atlet dan mahasiswa?\n",
    "\n",
    "Adakah kewujudan universiti sukan akan memberi impak besar kepada kemajuan sukan negara? Jika dirancang dan diatur dengan cekap dan sistematik, ia perkara tidak mustahil dicapai.\n",
    "\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'KUALA LUMPUR: Hakim Mahkamah Tinggi, Mohd Nazlan Mohd Ghazali menyifatkan kes penyelewengan dana RM42 juta milik SRC International Sdn Bhd dihadapi Datuk Seri Najib Razak adalah kesalahan salah guna kedudukan, pecah amanah jenayah dan pengubahan wang haram yang paling teruk. Mohd Nazlan yang mensabitkan Najib terhadap kesemua tujuh tuduhan dan memerintahkan bekas Perdana Menteri itu dipenjara 12 tahun, dan didenda RM210 juta, berkata ia bukan sahaja disebabkan oleh alasan bagaimana jenayah itu dilakukan, malah kes berprofil tinggi berkenaan turut membabitkan sejumlah wang yang sangat besar. Melalui alasan penghakiman penuh setebal 801 muka surat itu, Mohd Nazlan, berkata kes terbabit mempunyai elemen yang memberikan kesan ke atas kepentingan awam kerana dana RM42 juta itu adalah milik Kementerian Kewangan (Diperbadankan) (MKD) yang berkemungkinan berasal daripada dana pencen Kumpulan Wang Persaraan (Diperbadankan) (KWAP) berjumlah RM4 bilion. \"Dan yang paling penting ia membabitkan individu yang pada ketika itu berada dalam pada tertinggi dalam kerajaan,\" katanya. Pada 28 Julai lalu, Mohd Nazlan memerintahkan Najib dipenjarakan 10 tahun masing-masing bagi tiga tuduhan pecah amanah wang RM42 juta milik SRC. Hakim turut memerintahkan Najib dipenjara 12 tahun dan denda RM210 juta (jika gagal bayar, lima tahun penjara) bagi tuduhan menyalahgunakan kedudukan. Bagi tuduhan pengubahan wang haram pula, Mohd Nazlan memerintahkan Najib dipenjara 10 tahun bagi setiap tuduhan. Sementara itu, Mohd Nazlan berkata, Najib selaku tertuduh tidak menunjukkan penyesalan, malah mempertahankan pembelaan beliau tidak mengetahui mengenai wang RM42 juta milik SRC itu dalam rayuannya bagi diringankan hukuman. \"Tetapi saya tidak boleh menafikan beliau adalah Perdana Menteri negara ini dan tidak boleh mempersoalkan sumbangannya untuk kebaikan dan kesejahteraan masyarakat dalam pelbagai cara kerana beliau adalah Perdana Menteri selama sembilan tahun. \"Sejarah politik akan terus diperdebatkan sama ada dari segi keseimbangan, beliau melakukan lebih banyak kebaikan daripada keburukan. \"Walau apa pun, ia adalah tidak selari dengan idea sesebuah pentadbiran negara yang bersih daripada rasuah yang tidak boleh bertolak ansur dengan sebarang penyalahgunaan kuasa,\" katanya. Mahkamah Rayuan menetapkan pada 15 Oktober ini bagi pengurusan kes rayuan Najib terhadap sabitan dan hukuman terhadapnya.'"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cleaning(string)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "# splitted_string = malaya.text.function.split_into_sentences(transformer_textcleaning(string))\n",
    "# splitted_string2 = malaya.text.function.split_into_sentences(transformer_textcleaning(string2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "('Kes Najib salah guna dominasi Najib, undang wang haram',\n",
       " 'Kes SRC Najib salah guna dominasi Najib',\n",
       " 'Kes pecah amanah Najib salah guna dominasi pegawai negeri, kata hakim')"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "encoded = encoder.encode(f'tajuk: {transformer_textcleaning(string)}') + [1]\n",
    "f, b, n = sess.run([model.fast_result, model.beam_result, model.nucleus_result], \n",
    "             feed_dict = {model.X: [encoded], model.top_p: 0.7})\n",
    "\n",
    "(encoder.decode(f[0].tolist()), \n",
    "encoder.decode(b[0].tolist()), \n",
    "encoder.decode(n[0].tolist()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "('Kes SRC Najib salah guna dominasi Najib, pecah amanah jenayah',\n",
       " 'Kes SRC Najib salah guna dominasi Najib, pecah amanah jenayah',\n",
       " 'Kes Najib salah guna dominasi isytihar, kalah, kekal penjara')"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "encoded = encoder.encode(f'ringkasan: {transformer_textcleaning(string)}') + [1]\n",
    "f, b, n = sess.run([model.fast_result, model.beam_result, model.nucleus_result], \n",
    "             feed_dict = {model.X: [encoded], model.top_p: 0.7})\n",
    "\n",
    "(encoder.decode(f[0].tolist()), \n",
    "encoder.decode(b[0].tolist()), \n",
    "encoder.decode(n[0].tolist()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "('Dua calon PKR Sabah umum calon',\n",
       " 'Dua calon PKR Sabah umum calon',\n",
       " '200,000 calon PKR Sabah pada 2018')"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "encoded = encoder.encode(f'tajuk: {transformer_textcleaning(string2)}') + [1]\n",
    "f, b, n = sess.run([model.fast_result, model.beam_result, model.nucleus_result], \n",
    "             feed_dict = {model.X: [encoded], model.top_p: 0.7})\n",
    "\n",
    "(encoder.decode(f[0].tolist()), \n",
    "encoder.decode(b[0].tolist()), \n",
    "encoder.decode(n[0].tolist()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "('Kejayaan sukan lebih baik',\n",
       " 'Kejayaan sukan lebih baik',\n",
       " 'Kejayaan sukan seiringan bukan jaminan Sukan')"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "encoded = encoder.encode(f'tajuk: {transformer_textcleaning(string3)}') + [1]\n",
    "f, b, n = sess.run([model.fast_result, model.beam_result, model.nucleus_result], \n",
    "             feed_dict = {model.X: [encoded], model.top_p: 0.7})\n",
    "\n",
    "(encoder.decode(f[0].tolist()), \n",
    "encoder.decode(b[0].tolist()), \n",
    "encoder.decode(n[0].tolist()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "('Kejayaan sukan lebih baik',\n",
       " 'Kejayaan sukan lebih baik',\n",
       " 'Tigers dan PIL 2018; Ibu bapa cuai Kinninja Kal')"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "encoded = encoder.encode(f'ringkasan: {transformer_textcleaning(string3)}') + [1]\n",
    "f, b, n = sess.run([model.fast_result, model.beam_result, model.nucleus_result], \n",
    "             feed_dict = {model.X: [encoded], model.top_p: 0.7})\n",
    "\n",
    "(encoder.decode(f[0].tolist()), \n",
    "encoder.decode(b[0].tolist()), \n",
    "encoder.decode(n[0].tolist()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'transformer-small/model.ckpt'"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "saver = tf.train.Saver(tf.trainable_variables())\n",
    "saver.save(sess, 'transformer-small/model.ckpt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Placeholder',\n",
       " 'Placeholder_1',\n",
       " 'Placeholder_2',\n",
       " 'transformer/body/target_space_embedding/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/target_space_embedding/ReadVariableOp',\n",
       " 'transformer/body/encoder/layer_0/self_attention/layer_prepostprocess/layer_norm/layer_norm_scale/Read/ReadVariableOp',\n",
       " 'transformer/body/encoder/layer_0/self_attention/layer_prepostprocess/layer_norm/layer_norm_bias/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_0/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_0/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/body/encoder/layer_0/self_attention/multihead_attention/q/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_0/self_attention/multihead_attention/q/Tensordot/ReadVariableOp',\n",
       " 'transformer/body/encoder/layer_0/self_attention/multihead_attention/k/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_0/self_attention/multihead_attention/k/Tensordot/ReadVariableOp',\n",
       " 'transformer/body/encoder/layer_0/self_attention/multihead_attention/v/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_0/self_attention/multihead_attention/v/Tensordot/ReadVariableOp',\n",
       " 'transformer/body/encoder/layer_0/self_attention/multihead_attention/output_transform/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_0/self_attention/multihead_attention/output_transform/Tensordot/ReadVariableOp',\n",
       " 'transformer/body/encoder/layer_0/ffn/layer_prepostprocess/layer_norm/layer_norm_scale/Read/ReadVariableOp',\n",
       " 'transformer/body/encoder/layer_0/ffn/layer_prepostprocess/layer_norm/layer_norm_bias/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_0/ffn/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_0/ffn/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/body/encoder/layer_0/ffn/conv1/kernel/Read/ReadVariableOp',\n",
       " 'transformer/body/encoder/layer_0/ffn/conv1/bias/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_0/ffn/conv1/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_0/ffn/conv1/BiasAdd/ReadVariableOp',\n",
       " 'transformer/body/encoder/layer_0/ffn/conv2/kernel/Read/ReadVariableOp',\n",
       " 'transformer/body/encoder/layer_0/ffn/conv2/bias/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_0/ffn/conv2/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_0/ffn/conv2/BiasAdd/ReadVariableOp',\n",
       " 'transformer/body/encoder/layer_1/self_attention/layer_prepostprocess/layer_norm/layer_norm_scale/Read/ReadVariableOp',\n",
       " 'transformer/body/encoder/layer_1/self_attention/layer_prepostprocess/layer_norm/layer_norm_bias/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_1/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_1/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/body/encoder/layer_1/self_attention/multihead_attention/q/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_1/self_attention/multihead_attention/q/Tensordot/ReadVariableOp',\n",
       " 'transformer/body/encoder/layer_1/self_attention/multihead_attention/k/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_1/self_attention/multihead_attention/k/Tensordot/ReadVariableOp',\n",
       " 'transformer/body/encoder/layer_1/self_attention/multihead_attention/v/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_1/self_attention/multihead_attention/v/Tensordot/ReadVariableOp',\n",
       " 'transformer/body/encoder/layer_1/self_attention/multihead_attention/output_transform/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_1/self_attention/multihead_attention/output_transform/Tensordot/ReadVariableOp',\n",
       " 'transformer/body/encoder/layer_1/ffn/layer_prepostprocess/layer_norm/layer_norm_scale/Read/ReadVariableOp',\n",
       " 'transformer/body/encoder/layer_1/ffn/layer_prepostprocess/layer_norm/layer_norm_bias/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_1/ffn/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_1/ffn/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/body/encoder/layer_1/ffn/conv1/kernel/Read/ReadVariableOp',\n",
       " 'transformer/body/encoder/layer_1/ffn/conv1/bias/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_1/ffn/conv1/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_1/ffn/conv1/BiasAdd/ReadVariableOp',\n",
       " 'transformer/body/encoder/layer_1/ffn/conv2/kernel/Read/ReadVariableOp',\n",
       " 'transformer/body/encoder/layer_1/ffn/conv2/bias/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_1/ffn/conv2/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_1/ffn/conv2/BiasAdd/ReadVariableOp',\n",
       " 'transformer/body/encoder/layer_2/self_attention/layer_prepostprocess/layer_norm/layer_norm_scale/Read/ReadVariableOp',\n",
       " 'transformer/body/encoder/layer_2/self_attention/layer_prepostprocess/layer_norm/layer_norm_bias/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_2/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_2/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/body/encoder/layer_2/self_attention/multihead_attention/q/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_2/self_attention/multihead_attention/q/Tensordot/ReadVariableOp',\n",
       " 'transformer/body/encoder/layer_2/self_attention/multihead_attention/k/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_2/self_attention/multihead_attention/k/Tensordot/ReadVariableOp',\n",
       " 'transformer/body/encoder/layer_2/self_attention/multihead_attention/v/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_2/self_attention/multihead_attention/v/Tensordot/ReadVariableOp',\n",
       " 'transformer/body/encoder/layer_2/self_attention/multihead_attention/output_transform/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_2/self_attention/multihead_attention/output_transform/Tensordot/ReadVariableOp',\n",
       " 'transformer/body/encoder/layer_2/ffn/layer_prepostprocess/layer_norm/layer_norm_scale/Read/ReadVariableOp',\n",
       " 'transformer/body/encoder/layer_2/ffn/layer_prepostprocess/layer_norm/layer_norm_bias/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_2/ffn/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_2/ffn/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/body/encoder/layer_2/ffn/conv1/kernel/Read/ReadVariableOp',\n",
       " 'transformer/body/encoder/layer_2/ffn/conv1/bias/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_2/ffn/conv1/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_2/ffn/conv1/BiasAdd/ReadVariableOp',\n",
       " 'transformer/body/encoder/layer_2/ffn/conv2/kernel/Read/ReadVariableOp',\n",
       " 'transformer/body/encoder/layer_2/ffn/conv2/bias/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_2/ffn/conv2/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_2/ffn/conv2/BiasAdd/ReadVariableOp',\n",
       " 'transformer/body/encoder/layer_3/self_attention/layer_prepostprocess/layer_norm/layer_norm_scale/Read/ReadVariableOp',\n",
       " 'transformer/body/encoder/layer_3/self_attention/layer_prepostprocess/layer_norm/layer_norm_bias/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_3/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_3/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/body/encoder/layer_3/self_attention/multihead_attention/q/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_3/self_attention/multihead_attention/q/Tensordot/ReadVariableOp',\n",
       " 'transformer/body/encoder/layer_3/self_attention/multihead_attention/k/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_3/self_attention/multihead_attention/k/Tensordot/ReadVariableOp',\n",
       " 'transformer/body/encoder/layer_3/self_attention/multihead_attention/v/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_3/self_attention/multihead_attention/v/Tensordot/ReadVariableOp',\n",
       " 'transformer/body/encoder/layer_3/self_attention/multihead_attention/output_transform/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_3/self_attention/multihead_attention/output_transform/Tensordot/ReadVariableOp',\n",
       " 'transformer/body/encoder/layer_3/ffn/layer_prepostprocess/layer_norm/layer_norm_scale/Read/ReadVariableOp',\n",
       " 'transformer/body/encoder/layer_3/ffn/layer_prepostprocess/layer_norm/layer_norm_bias/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_3/ffn/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_3/ffn/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/body/encoder/layer_3/ffn/conv1/kernel/Read/ReadVariableOp',\n",
       " 'transformer/body/encoder/layer_3/ffn/conv1/bias/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_3/ffn/conv1/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_3/ffn/conv1/BiasAdd/ReadVariableOp',\n",
       " 'transformer/body/encoder/layer_3/ffn/conv2/kernel/Read/ReadVariableOp',\n",
       " 'transformer/body/encoder/layer_3/ffn/conv2/bias/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_3/ffn/conv2/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_3/ffn/conv2/BiasAdd/ReadVariableOp',\n",
       " 'transformer/body/encoder/layer_4/self_attention/layer_prepostprocess/layer_norm/layer_norm_scale/Read/ReadVariableOp',\n",
       " 'transformer/body/encoder/layer_4/self_attention/layer_prepostprocess/layer_norm/layer_norm_bias/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_4/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_4/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/body/encoder/layer_4/self_attention/multihead_attention/q/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_4/self_attention/multihead_attention/q/Tensordot/ReadVariableOp',\n",
       " 'transformer/body/encoder/layer_4/self_attention/multihead_attention/k/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_4/self_attention/multihead_attention/k/Tensordot/ReadVariableOp',\n",
       " 'transformer/body/encoder/layer_4/self_attention/multihead_attention/v/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_4/self_attention/multihead_attention/v/Tensordot/ReadVariableOp',\n",
       " 'transformer/body/encoder/layer_4/self_attention/multihead_attention/output_transform/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_4/self_attention/multihead_attention/output_transform/Tensordot/ReadVariableOp',\n",
       " 'transformer/body/encoder/layer_4/ffn/layer_prepostprocess/layer_norm/layer_norm_scale/Read/ReadVariableOp',\n",
       " 'transformer/body/encoder/layer_4/ffn/layer_prepostprocess/layer_norm/layer_norm_bias/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_4/ffn/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_4/ffn/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/body/encoder/layer_4/ffn/conv1/kernel/Read/ReadVariableOp',\n",
       " 'transformer/body/encoder/layer_4/ffn/conv1/bias/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_4/ffn/conv1/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_4/ffn/conv1/BiasAdd/ReadVariableOp',\n",
       " 'transformer/body/encoder/layer_4/ffn/conv2/kernel/Read/ReadVariableOp',\n",
       " 'transformer/body/encoder/layer_4/ffn/conv2/bias/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_4/ffn/conv2/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_4/ffn/conv2/BiasAdd/ReadVariableOp',\n",
       " 'transformer/body/encoder/layer_5/self_attention/layer_prepostprocess/layer_norm/layer_norm_scale/Read/ReadVariableOp',\n",
       " 'transformer/body/encoder/layer_5/self_attention/layer_prepostprocess/layer_norm/layer_norm_bias/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_5/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_5/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/body/encoder/layer_5/self_attention/multihead_attention/q/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_5/self_attention/multihead_attention/q/Tensordot/ReadVariableOp',\n",
       " 'transformer/body/encoder/layer_5/self_attention/multihead_attention/k/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_5/self_attention/multihead_attention/k/Tensordot/ReadVariableOp',\n",
       " 'transformer/body/encoder/layer_5/self_attention/multihead_attention/v/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_5/self_attention/multihead_attention/v/Tensordot/ReadVariableOp',\n",
       " 'transformer/body/encoder/layer_5/self_attention/multihead_attention/output_transform/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_5/self_attention/multihead_attention/output_transform/Tensordot/ReadVariableOp',\n",
       " 'transformer/body/encoder/layer_5/ffn/layer_prepostprocess/layer_norm/layer_norm_scale/Read/ReadVariableOp',\n",
       " 'transformer/body/encoder/layer_5/ffn/layer_prepostprocess/layer_norm/layer_norm_bias/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_5/ffn/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_5/ffn/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/body/encoder/layer_5/ffn/conv1/kernel/Read/ReadVariableOp',\n",
       " 'transformer/body/encoder/layer_5/ffn/conv1/bias/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_5/ffn/conv1/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_5/ffn/conv1/BiasAdd/ReadVariableOp',\n",
       " 'transformer/body/encoder/layer_5/ffn/conv2/kernel/Read/ReadVariableOp',\n",
       " 'transformer/body/encoder/layer_5/ffn/conv2/bias/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_5/ffn/conv2/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_5/ffn/conv2/BiasAdd/ReadVariableOp',\n",
       " 'transformer/body/encoder/layer_prepostprocess/layer_norm/layer_norm_scale/Read/ReadVariableOp',\n",
       " 'transformer/body/encoder/layer_prepostprocess/layer_norm/layer_norm_bias/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/body/decoder/layer_0/self_attention/layer_prepostprocess/layer_norm/layer_norm_scale/Read/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_0/self_attention/layer_prepostprocess/layer_norm/layer_norm_bias/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_0/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_0/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/body/decoder/layer_0/self_attention/multihead_attention/q/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_0/self_attention/multihead_attention/q/Tensordot/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_0/self_attention/multihead_attention/k/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_0/self_attention/multihead_attention/k/Tensordot/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_0/self_attention/multihead_attention/v/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_0/self_attention/multihead_attention/v/Tensordot/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_0/self_attention/multihead_attention/output_transform/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_0/self_attention/multihead_attention/output_transform/Tensordot/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_0/encdec_attention/layer_prepostprocess/layer_norm/layer_norm_scale/Read/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_0/encdec_attention/layer_prepostprocess/layer_norm/layer_norm_bias/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_0/encdec_attention/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_0/encdec_attention/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/body/decoder/layer_0/encdec_attention/multihead_attention/q/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_0/encdec_attention/multihead_attention/q/Tensordot/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_0/encdec_attention/multihead_attention/k/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_0/encdec_attention/multihead_attention/k/Tensordot/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_0/encdec_attention/multihead_attention/v/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_0/encdec_attention/multihead_attention/v/Tensordot/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_0/encdec_attention/multihead_attention/output_transform/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_0/encdec_attention/multihead_attention/output_transform/Tensordot/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_0/ffn/layer_prepostprocess/layer_norm/layer_norm_scale/Read/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_0/ffn/layer_prepostprocess/layer_norm/layer_norm_bias/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_0_1/ffn/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_0_1/ffn/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/body/decoder/layer_0/ffn/conv1/kernel/Read/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_0/ffn/conv1/bias/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_0_1/ffn/conv1/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_0_1/ffn/conv1/BiasAdd/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_0/ffn/conv2/kernel/Read/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_0/ffn/conv2/bias/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_0_1/ffn/conv2/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_0_1/ffn/conv2/BiasAdd/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_1/self_attention/layer_prepostprocess/layer_norm/layer_norm_scale/Read/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_1/self_attention/layer_prepostprocess/layer_norm/layer_norm_bias/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_1/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_1/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/body/decoder/layer_1/self_attention/multihead_attention/q/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_1/self_attention/multihead_attention/q/Tensordot/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_1/self_attention/multihead_attention/k/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_1/self_attention/multihead_attention/k/Tensordot/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_1/self_attention/multihead_attention/v/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_1/self_attention/multihead_attention/v/Tensordot/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_1/self_attention/multihead_attention/output_transform/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_1/self_attention/multihead_attention/output_transform/Tensordot/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_1/encdec_attention/layer_prepostprocess/layer_norm/layer_norm_scale/Read/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_1/encdec_attention/layer_prepostprocess/layer_norm/layer_norm_bias/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_1/encdec_attention/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_1/encdec_attention/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/body/decoder/layer_1/encdec_attention/multihead_attention/q/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_1/encdec_attention/multihead_attention/q/Tensordot/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_1/encdec_attention/multihead_attention/k/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_1/encdec_attention/multihead_attention/k/Tensordot/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_1/encdec_attention/multihead_attention/v/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_1/encdec_attention/multihead_attention/v/Tensordot/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_1/encdec_attention/multihead_attention/output_transform/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_1/encdec_attention/multihead_attention/output_transform/Tensordot/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_1/ffn/layer_prepostprocess/layer_norm/layer_norm_scale/Read/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_1/ffn/layer_prepostprocess/layer_norm/layer_norm_bias/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_1_1/ffn/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_1_1/ffn/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/body/decoder/layer_1/ffn/conv1/kernel/Read/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_1/ffn/conv1/bias/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_1_1/ffn/conv1/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_1_1/ffn/conv1/BiasAdd/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_1/ffn/conv2/kernel/Read/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_1/ffn/conv2/bias/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_1_1/ffn/conv2/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_1_1/ffn/conv2/BiasAdd/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_2/self_attention/layer_prepostprocess/layer_norm/layer_norm_scale/Read/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_2/self_attention/layer_prepostprocess/layer_norm/layer_norm_bias/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_2/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_2/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/body/decoder/layer_2/self_attention/multihead_attention/q/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_2/self_attention/multihead_attention/q/Tensordot/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_2/self_attention/multihead_attention/k/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_2/self_attention/multihead_attention/k/Tensordot/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_2/self_attention/multihead_attention/v/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_2/self_attention/multihead_attention/v/Tensordot/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_2/self_attention/multihead_attention/output_transform/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_2/self_attention/multihead_attention/output_transform/Tensordot/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_2/encdec_attention/layer_prepostprocess/layer_norm/layer_norm_scale/Read/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_2/encdec_attention/layer_prepostprocess/layer_norm/layer_norm_bias/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_2/encdec_attention/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_2/encdec_attention/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/body/decoder/layer_2/encdec_attention/multihead_attention/q/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_2/encdec_attention/multihead_attention/q/Tensordot/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_2/encdec_attention/multihead_attention/k/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_2/encdec_attention/multihead_attention/k/Tensordot/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_2/encdec_attention/multihead_attention/v/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_2/encdec_attention/multihead_attention/v/Tensordot/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_2/encdec_attention/multihead_attention/output_transform/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_2/encdec_attention/multihead_attention/output_transform/Tensordot/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_2/ffn/layer_prepostprocess/layer_norm/layer_norm_scale/Read/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_2/ffn/layer_prepostprocess/layer_norm/layer_norm_bias/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_2_1/ffn/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_2_1/ffn/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/body/decoder/layer_2/ffn/conv1/kernel/Read/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_2/ffn/conv1/bias/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_2_1/ffn/conv1/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_2_1/ffn/conv1/BiasAdd/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_2/ffn/conv2/kernel/Read/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_2/ffn/conv2/bias/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_2_1/ffn/conv2/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_2_1/ffn/conv2/BiasAdd/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_3/self_attention/layer_prepostprocess/layer_norm/layer_norm_scale/Read/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_3/self_attention/layer_prepostprocess/layer_norm/layer_norm_bias/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_3/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_3/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/body/decoder/layer_3/self_attention/multihead_attention/q/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_3/self_attention/multihead_attention/q/Tensordot/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_3/self_attention/multihead_attention/k/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_3/self_attention/multihead_attention/k/Tensordot/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_3/self_attention/multihead_attention/v/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_3/self_attention/multihead_attention/v/Tensordot/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_3/self_attention/multihead_attention/output_transform/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_3/self_attention/multihead_attention/output_transform/Tensordot/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_3/encdec_attention/layer_prepostprocess/layer_norm/layer_norm_scale/Read/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_3/encdec_attention/layer_prepostprocess/layer_norm/layer_norm_bias/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_3/encdec_attention/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_3/encdec_attention/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/body/decoder/layer_3/encdec_attention/multihead_attention/q/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_3/encdec_attention/multihead_attention/q/Tensordot/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_3/encdec_attention/multihead_attention/k/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_3/encdec_attention/multihead_attention/k/Tensordot/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_3/encdec_attention/multihead_attention/v/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_3/encdec_attention/multihead_attention/v/Tensordot/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_3/encdec_attention/multihead_attention/output_transform/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_3/encdec_attention/multihead_attention/output_transform/Tensordot/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_3/ffn/layer_prepostprocess/layer_norm/layer_norm_scale/Read/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_3/ffn/layer_prepostprocess/layer_norm/layer_norm_bias/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_3_1/ffn/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_3_1/ffn/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/body/decoder/layer_3/ffn/conv1/kernel/Read/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_3/ffn/conv1/bias/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_3_1/ffn/conv1/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_3_1/ffn/conv1/BiasAdd/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_3/ffn/conv2/kernel/Read/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_3/ffn/conv2/bias/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_3_1/ffn/conv2/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_3_1/ffn/conv2/BiasAdd/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_4/self_attention/layer_prepostprocess/layer_norm/layer_norm_scale/Read/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_4/self_attention/layer_prepostprocess/layer_norm/layer_norm_bias/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_4/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_4/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/body/decoder/layer_4/self_attention/multihead_attention/q/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_4/self_attention/multihead_attention/q/Tensordot/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_4/self_attention/multihead_attention/k/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_4/self_attention/multihead_attention/k/Tensordot/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_4/self_attention/multihead_attention/v/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_4/self_attention/multihead_attention/v/Tensordot/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_4/self_attention/multihead_attention/output_transform/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_4/self_attention/multihead_attention/output_transform/Tensordot/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_4/encdec_attention/layer_prepostprocess/layer_norm/layer_norm_scale/Read/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_4/encdec_attention/layer_prepostprocess/layer_norm/layer_norm_bias/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_4/encdec_attention/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_4/encdec_attention/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/body/decoder/layer_4/encdec_attention/multihead_attention/q/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_4/encdec_attention/multihead_attention/q/Tensordot/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_4/encdec_attention/multihead_attention/k/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_4/encdec_attention/multihead_attention/k/Tensordot/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_4/encdec_attention/multihead_attention/v/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_4/encdec_attention/multihead_attention/v/Tensordot/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_4/encdec_attention/multihead_attention/output_transform/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_4/encdec_attention/multihead_attention/output_transform/Tensordot/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_4/ffn/layer_prepostprocess/layer_norm/layer_norm_scale/Read/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_4/ffn/layer_prepostprocess/layer_norm/layer_norm_bias/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_4_1/ffn/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_4_1/ffn/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/body/decoder/layer_4/ffn/conv1/kernel/Read/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_4/ffn/conv1/bias/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_4_1/ffn/conv1/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_4_1/ffn/conv1/BiasAdd/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_4/ffn/conv2/kernel/Read/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_4/ffn/conv2/bias/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_4_1/ffn/conv2/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_4_1/ffn/conv2/BiasAdd/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_5/self_attention/layer_prepostprocess/layer_norm/layer_norm_scale/Read/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_5/self_attention/layer_prepostprocess/layer_norm/layer_norm_bias/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_5/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_5/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/body/decoder/layer_5/self_attention/multihead_attention/q/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_5/self_attention/multihead_attention/q/Tensordot/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_5/self_attention/multihead_attention/k/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_5/self_attention/multihead_attention/k/Tensordot/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_5/self_attention/multihead_attention/v/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_5/self_attention/multihead_attention/v/Tensordot/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_5/self_attention/multihead_attention/output_transform/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_5/self_attention/multihead_attention/output_transform/Tensordot/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_5/encdec_attention/layer_prepostprocess/layer_norm/layer_norm_scale/Read/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_5/encdec_attention/layer_prepostprocess/layer_norm/layer_norm_bias/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_5/encdec_attention/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_5/encdec_attention/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/body/decoder/layer_5/encdec_attention/multihead_attention/q/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_5/encdec_attention/multihead_attention/q/Tensordot/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_5/encdec_attention/multihead_attention/k/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_5/encdec_attention/multihead_attention/k/Tensordot/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_5/encdec_attention/multihead_attention/v/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_5/encdec_attention/multihead_attention/v/Tensordot/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_5/encdec_attention/multihead_attention/output_transform/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_5/encdec_attention/multihead_attention/output_transform/Tensordot/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_5/ffn/layer_prepostprocess/layer_norm/layer_norm_scale/Read/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_5/ffn/layer_prepostprocess/layer_norm/layer_norm_bias/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_5_1/ffn/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_5_1/ffn/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/body/decoder/layer_5/ffn/conv1/kernel/Read/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_5/ffn/conv1/bias/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_5_1/ffn/conv1/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_5_1/ffn/conv1/BiasAdd/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_5/ffn/conv2/kernel/Read/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_5/ffn/conv2/bias/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_5_1/ffn/conv2/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_5_1/ffn/conv2/BiasAdd/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_prepostprocess/layer_norm/layer_norm_scale/Read/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_prepostprocess/layer_norm/layer_norm_bias/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer_1/body/parallel_0/body/target_space_embedding/ReadVariableOp',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_0/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_0/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_0/self_attention/multihead_attention/q/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_0/self_attention/multihead_attention/k/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_0/self_attention/multihead_attention/v/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_0/self_attention/multihead_attention/output_transform/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_0/ffn/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_0/ffn/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_0/ffn/conv1/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_0/ffn/conv1/BiasAdd/ReadVariableOp',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_0/ffn/conv2/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_0/ffn/conv2/BiasAdd/ReadVariableOp',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_1/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_1/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_1/self_attention/multihead_attention/q/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_1/self_attention/multihead_attention/k/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_1/self_attention/multihead_attention/v/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_1/self_attention/multihead_attention/output_transform/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_1/ffn/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_1/ffn/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_1/ffn/conv1/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_1/ffn/conv1/BiasAdd/ReadVariableOp',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_1/ffn/conv2/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_1/ffn/conv2/BiasAdd/ReadVariableOp',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_2/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_2/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_2/self_attention/multihead_attention/q/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_2/self_attention/multihead_attention/k/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_2/self_attention/multihead_attention/v/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_2/self_attention/multihead_attention/output_transform/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_2/ffn/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_2/ffn/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_2/ffn/conv1/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_2/ffn/conv1/BiasAdd/ReadVariableOp',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_2/ffn/conv2/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_2/ffn/conv2/BiasAdd/ReadVariableOp',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_3/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_3/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_3/self_attention/multihead_attention/q/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_3/self_attention/multihead_attention/k/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_3/self_attention/multihead_attention/v/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_3/self_attention/multihead_attention/output_transform/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_3/ffn/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_3/ffn/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_3/ffn/conv1/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_3/ffn/conv1/BiasAdd/ReadVariableOp',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_3/ffn/conv2/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_3/ffn/conv2/BiasAdd/ReadVariableOp',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_4/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_4/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_4/self_attention/multihead_attention/q/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_4/self_attention/multihead_attention/k/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_4/self_attention/multihead_attention/v/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_4/self_attention/multihead_attention/output_transform/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_4/ffn/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_4/ffn/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_4/ffn/conv1/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_4/ffn/conv1/BiasAdd/ReadVariableOp',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_4/ffn/conv2/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_4/ffn/conv2/BiasAdd/ReadVariableOp',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_5/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_5/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_5/self_attention/multihead_attention/q/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_5/self_attention/multihead_attention/k/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_5/self_attention/multihead_attention/v/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_5/self_attention/multihead_attention/output_transform/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_5/ffn/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_5/ffn/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_5/ffn/conv1/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_5/ffn/conv1/BiasAdd/ReadVariableOp',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_5/ffn/conv2/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_5/ffn/conv2/BiasAdd/ReadVariableOp',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer_1/body/decoder/layer_0/encdec_attention/multihead_attention/k/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/body/decoder/layer_0/encdec_attention/multihead_attention/v/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/body/decoder/layer_1/encdec_attention/multihead_attention/k/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/body/decoder/layer_1/encdec_attention/multihead_attention/v/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/body/decoder/layer_2/encdec_attention/multihead_attention/k/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/body/decoder/layer_2/encdec_attention/multihead_attention/v/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/body/decoder/layer_3/encdec_attention/multihead_attention/k/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/body/decoder/layer_3/encdec_attention/multihead_attention/v/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/body/decoder/layer_4/encdec_attention/multihead_attention/k/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/body/decoder/layer_4/encdec_attention/multihead_attention/v/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/body/decoder/layer_5/encdec_attention/multihead_attention/k/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/body/decoder/layer_5/encdec_attention/multihead_attention/v/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_0/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_0/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_0/self_attention/multihead_attention/q/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_0/self_attention/multihead_attention/k/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_0/self_attention/multihead_attention/v/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_0/self_attention/multihead_attention/output_transform/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_0/encdec_attention/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_0/encdec_attention/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_0/encdec_attention/multihead_attention/q/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_0/encdec_attention/multihead_attention/output_transform/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_0_1/ffn/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_0_1/ffn/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_0_1/ffn/conv1/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_0_1/ffn/conv1/BiasAdd/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_0_1/ffn/conv2/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_0_1/ffn/conv2/BiasAdd/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_1/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_1/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_1/self_attention/multihead_attention/q/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_1/self_attention/multihead_attention/k/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_1/self_attention/multihead_attention/v/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_1/self_attention/multihead_attention/output_transform/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_1/encdec_attention/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_1/encdec_attention/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_1/encdec_attention/multihead_attention/q/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_1/encdec_attention/multihead_attention/output_transform/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_1_1/ffn/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_1_1/ffn/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_1_1/ffn/conv1/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_1_1/ffn/conv1/BiasAdd/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_1_1/ffn/conv2/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_1_1/ffn/conv2/BiasAdd/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_2/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_2/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_2/self_attention/multihead_attention/q/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_2/self_attention/multihead_attention/k/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_2/self_attention/multihead_attention/v/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_2/self_attention/multihead_attention/output_transform/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_2/encdec_attention/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_2/encdec_attention/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_2/encdec_attention/multihead_attention/q/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_2/encdec_attention/multihead_attention/output_transform/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_2_1/ffn/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_2_1/ffn/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_2_1/ffn/conv1/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_2_1/ffn/conv1/BiasAdd/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_2_1/ffn/conv2/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_2_1/ffn/conv2/BiasAdd/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_3/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_3/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_3/self_attention/multihead_attention/q/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_3/self_attention/multihead_attention/k/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_3/self_attention/multihead_attention/v/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_3/self_attention/multihead_attention/output_transform/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_3/encdec_attention/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_3/encdec_attention/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_3/encdec_attention/multihead_attention/q/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_3/encdec_attention/multihead_attention/output_transform/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_3_1/ffn/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_3_1/ffn/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_3_1/ffn/conv1/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_3_1/ffn/conv1/BiasAdd/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_3_1/ffn/conv2/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_3_1/ffn/conv2/BiasAdd/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_4/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_4/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_4/self_attention/multihead_attention/q/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_4/self_attention/multihead_attention/k/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_4/self_attention/multihead_attention/v/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_4/self_attention/multihead_attention/output_transform/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_4/encdec_attention/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_4/encdec_attention/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_4/encdec_attention/multihead_attention/q/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_4/encdec_attention/multihead_attention/output_transform/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_4_1/ffn/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_4_1/ffn/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_4_1/ffn/conv1/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_4_1/ffn/conv1/BiasAdd/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_4_1/ffn/conv2/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_4_1/ffn/conv2/BiasAdd/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_5/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_5/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_5/self_attention/multihead_attention/q/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_5/self_attention/multihead_attention/k/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_5/self_attention/multihead_attention/v/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_5/self_attention/multihead_attention/output_transform/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_5/encdec_attention/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_5/encdec_attention/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_5/encdec_attention/multihead_attention/q/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_5/encdec_attention/multihead_attention/output_transform/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_5_1/ffn/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_5_1/ffn/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_5_1/ffn/conv1/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_5_1/ffn/conv1/BiasAdd/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_5_1/ffn/conv2/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_5_1/ffn/conv2/BiasAdd/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/target_space_embedding/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_0/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_0/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_0/self_attention/multihead_attention/q/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_0/self_attention/multihead_attention/k/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_0/self_attention/multihead_attention/v/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_0/self_attention/multihead_attention/output_transform/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_0/ffn/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_0/ffn/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_0/ffn/conv1/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_0/ffn/conv1/BiasAdd/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_0/ffn/conv2/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_0/ffn/conv2/BiasAdd/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_1/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_1/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_1/self_attention/multihead_attention/q/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_1/self_attention/multihead_attention/k/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_1/self_attention/multihead_attention/v/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_1/self_attention/multihead_attention/output_transform/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_1/ffn/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_1/ffn/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_1/ffn/conv1/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_1/ffn/conv1/BiasAdd/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_1/ffn/conv2/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_1/ffn/conv2/BiasAdd/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_2/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_2/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_2/self_attention/multihead_attention/q/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_2/self_attention/multihead_attention/k/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_2/self_attention/multihead_attention/v/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_2/self_attention/multihead_attention/output_transform/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_2/ffn/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_2/ffn/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_2/ffn/conv1/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_2/ffn/conv1/BiasAdd/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_2/ffn/conv2/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_2/ffn/conv2/BiasAdd/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_3/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_3/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_3/self_attention/multihead_attention/q/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_3/self_attention/multihead_attention/k/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_3/self_attention/multihead_attention/v/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_3/self_attention/multihead_attention/output_transform/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_3/ffn/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_3/ffn/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_3/ffn/conv1/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_3/ffn/conv1/BiasAdd/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_3/ffn/conv2/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_3/ffn/conv2/BiasAdd/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_4/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_4/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_4/self_attention/multihead_attention/q/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_4/self_attention/multihead_attention/k/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_4/self_attention/multihead_attention/v/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_4/self_attention/multihead_attention/output_transform/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_4/ffn/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_4/ffn/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_4/ffn/conv1/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_4/ffn/conv1/BiasAdd/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_4/ffn/conv2/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_4/ffn/conv2/BiasAdd/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_5/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_5/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_5/self_attention/multihead_attention/q/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_5/self_attention/multihead_attention/k/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_5/self_attention/multihead_attention/v/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_5/self_attention/multihead_attention/output_transform/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_5/ffn/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_5/ffn/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_5/ffn/conv1/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_5/ffn/conv1/BiasAdd/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_5/ffn/conv2/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_5/ffn/conv2/BiasAdd/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_0/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_0/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_0/self_attention/multihead_attention/q/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_0/self_attention/multihead_attention/k/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_0/self_attention/multihead_attention/v/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_0/self_attention/multihead_attention/output_transform/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_0/encdec_attention/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_0/encdec_attention/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_0/encdec_attention/multihead_attention/q/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_0/encdec_attention/multihead_attention/k/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_0/encdec_attention/multihead_attention/v/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_0/encdec_attention/multihead_attention/output_transform/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_0_1/ffn/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_0_1/ffn/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_0_1/ffn/conv1/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_0_1/ffn/conv1/BiasAdd/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_0_1/ffn/conv2/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_0_1/ffn/conv2/BiasAdd/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_1/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_1/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_1/self_attention/multihead_attention/q/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_1/self_attention/multihead_attention/k/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_1/self_attention/multihead_attention/v/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_1/self_attention/multihead_attention/output_transform/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_1/encdec_attention/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_1/encdec_attention/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_1/encdec_attention/multihead_attention/q/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_1/encdec_attention/multihead_attention/k/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_1/encdec_attention/multihead_attention/v/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_1/encdec_attention/multihead_attention/output_transform/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_1_1/ffn/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_1_1/ffn/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_1_1/ffn/conv1/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_1_1/ffn/conv1/BiasAdd/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_1_1/ffn/conv2/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_1_1/ffn/conv2/BiasAdd/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_2/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_2/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_2/self_attention/multihead_attention/q/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_2/self_attention/multihead_attention/k/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_2/self_attention/multihead_attention/v/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_2/self_attention/multihead_attention/output_transform/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_2/encdec_attention/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_2/encdec_attention/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_2/encdec_attention/multihead_attention/q/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_2/encdec_attention/multihead_attention/k/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_2/encdec_attention/multihead_attention/v/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_2/encdec_attention/multihead_attention/output_transform/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_2_1/ffn/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_2_1/ffn/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_2_1/ffn/conv1/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_2_1/ffn/conv1/BiasAdd/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_2_1/ffn/conv2/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_2_1/ffn/conv2/BiasAdd/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_3/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_3/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_3/self_attention/multihead_attention/q/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_3/self_attention/multihead_attention/k/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_3/self_attention/multihead_attention/v/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_3/self_attention/multihead_attention/output_transform/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_3/encdec_attention/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_3/encdec_attention/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_3/encdec_attention/multihead_attention/q/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_3/encdec_attention/multihead_attention/k/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_3/encdec_attention/multihead_attention/v/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_3/encdec_attention/multihead_attention/output_transform/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_3_1/ffn/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_3_1/ffn/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_3_1/ffn/conv1/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_3_1/ffn/conv1/BiasAdd/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_3_1/ffn/conv2/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_3_1/ffn/conv2/BiasAdd/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_4/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_4/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_4/self_attention/multihead_attention/q/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_4/self_attention/multihead_attention/k/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_4/self_attention/multihead_attention/v/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_4/self_attention/multihead_attention/output_transform/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_4/encdec_attention/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_4/encdec_attention/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_4/encdec_attention/multihead_attention/q/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_4/encdec_attention/multihead_attention/k/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_4/encdec_attention/multihead_attention/v/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_4/encdec_attention/multihead_attention/output_transform/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_4_1/ffn/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_4_1/ffn/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_4_1/ffn/conv1/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_4_1/ffn/conv1/BiasAdd/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_4_1/ffn/conv2/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_4_1/ffn/conv2/BiasAdd/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_5/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_5/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_5/self_attention/multihead_attention/q/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_5/self_attention/multihead_attention/k/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_5/self_attention/multihead_attention/v/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_5/self_attention/multihead_attention/output_transform/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_5/encdec_attention/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_5/encdec_attention/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_5/encdec_attention/multihead_attention/q/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_5/encdec_attention/multihead_attention/k/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_5/encdec_attention/multihead_attention/v/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_5/encdec_attention/multihead_attention/output_transform/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_5_1/ffn/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_5_1/ffn/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_5_1/ffn/conv1/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_5_1/ffn/conv1/BiasAdd/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_5_1/ffn/conv2/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_5_1/ffn/conv2/BiasAdd/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/parallel_0_16/transformer/transformer/body/target_space_embedding/ReadVariableOp',\n",
       " 'transformer/parallel_0_16/transformer/transformer/body/encoder/layer_0/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_16/transformer/transformer/body/encoder/layer_0/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/parallel_0_16/transformer/transformer/body/encoder/layer_0/self_attention/multihead_attention/q/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_16/transformer/transformer/body/encoder/layer_0/self_attention/multihead_attention/k/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_16/transformer/transformer/body/encoder/layer_0/self_attention/multihead_attention/v/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_16/transformer/transformer/body/encoder/layer_0/self_attention/multihead_attention/output_transform/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_16/transformer/transformer/body/encoder/layer_0/ffn/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_16/transformer/transformer/body/encoder/layer_0/ffn/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/parallel_0_16/transformer/transformer/body/encoder/layer_0/ffn/conv1/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_16/transformer/transformer/body/encoder/layer_0/ffn/conv1/BiasAdd/ReadVariableOp',\n",
       " 'transformer/parallel_0_16/transformer/transformer/body/encoder/layer_0/ffn/conv2/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_16/transformer/transformer/body/encoder/layer_0/ffn/conv2/BiasAdd/ReadVariableOp',\n",
       " 'transformer/parallel_0_16/transformer/transformer/body/encoder/layer_1/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_16/transformer/transformer/body/encoder/layer_1/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/parallel_0_16/transformer/transformer/body/encoder/layer_1/self_attention/multihead_attention/q/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_16/transformer/transformer/body/encoder/layer_1/self_attention/multihead_attention/k/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_16/transformer/transformer/body/encoder/layer_1/self_attention/multihead_attention/v/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_16/transformer/transformer/body/encoder/layer_1/self_attention/multihead_attention/output_transform/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_16/transformer/transformer/body/encoder/layer_1/ffn/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_16/transformer/transformer/body/encoder/layer_1/ffn/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/parallel_0_16/transformer/transformer/body/encoder/layer_1/ffn/conv1/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_16/transformer/transformer/body/encoder/layer_1/ffn/conv1/BiasAdd/ReadVariableOp',\n",
       " 'transformer/parallel_0_16/transformer/transformer/body/encoder/layer_1/ffn/conv2/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_16/transformer/transformer/body/encoder/layer_1/ffn/conv2/BiasAdd/ReadVariableOp',\n",
       " 'transformer/parallel_0_16/transformer/transformer/body/encoder/layer_2/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_16/transformer/transformer/body/encoder/layer_2/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/parallel_0_16/transformer/transformer/body/encoder/layer_2/self_attention/multihead_attention/q/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_16/transformer/transformer/body/encoder/layer_2/self_attention/multihead_attention/k/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_16/transformer/transformer/body/encoder/layer_2/self_attention/multihead_attention/v/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_16/transformer/transformer/body/encoder/layer_2/self_attention/multihead_attention/output_transform/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_16/transformer/transformer/body/encoder/layer_2/ffn/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_16/transformer/transformer/body/encoder/layer_2/ffn/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/parallel_0_16/transformer/transformer/body/encoder/layer_2/ffn/conv1/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_16/transformer/transformer/body/encoder/layer_2/ffn/conv1/BiasAdd/ReadVariableOp',\n",
       " 'transformer/parallel_0_16/transformer/transformer/body/encoder/layer_2/ffn/conv2/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_16/transformer/transformer/body/encoder/layer_2/ffn/conv2/BiasAdd/ReadVariableOp',\n",
       " 'transformer/parallel_0_16/transformer/transformer/body/encoder/layer_3/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_16/transformer/transformer/body/encoder/layer_3/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/parallel_0_16/transformer/transformer/body/encoder/layer_3/self_attention/multihead_attention/q/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_16/transformer/transformer/body/encoder/layer_3/self_attention/multihead_attention/k/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_16/transformer/transformer/body/encoder/layer_3/self_attention/multihead_attention/v/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_16/transformer/transformer/body/encoder/layer_3/self_attention/multihead_attention/output_transform/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_16/transformer/transformer/body/encoder/layer_3/ffn/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_16/transformer/transformer/body/encoder/layer_3/ffn/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/parallel_0_16/transformer/transformer/body/encoder/layer_3/ffn/conv1/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_16/transformer/transformer/body/encoder/layer_3/ffn/conv1/BiasAdd/ReadVariableOp',\n",
       " 'transformer/parallel_0_16/transformer/transformer/body/encoder/layer_3/ffn/conv2/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_16/transformer/transformer/body/encoder/layer_3/ffn/conv2/BiasAdd/ReadVariableOp',\n",
       " 'transformer/parallel_0_16/transformer/transformer/body/encoder/layer_4/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_16/transformer/transformer/body/encoder/layer_4/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/parallel_0_16/transformer/transformer/body/encoder/layer_4/self_attention/multihead_attention/q/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_16/transformer/transformer/body/encoder/layer_4/self_attention/multihead_attention/k/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_16/transformer/transformer/body/encoder/layer_4/self_attention/multihead_attention/v/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_16/transformer/transformer/body/encoder/layer_4/self_attention/multihead_attention/output_transform/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_16/transformer/transformer/body/encoder/layer_4/ffn/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_16/transformer/transformer/body/encoder/layer_4/ffn/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/parallel_0_16/transformer/transformer/body/encoder/layer_4/ffn/conv1/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_16/transformer/transformer/body/encoder/layer_4/ffn/conv1/BiasAdd/ReadVariableOp',\n",
       " 'transformer/parallel_0_16/transformer/transformer/body/encoder/layer_4/ffn/conv2/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_16/transformer/transformer/body/encoder/layer_4/ffn/conv2/BiasAdd/ReadVariableOp',\n",
       " 'transformer/parallel_0_16/transformer/transformer/body/encoder/layer_5/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_16/transformer/transformer/body/encoder/layer_5/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/parallel_0_16/transformer/transformer/body/encoder/layer_5/self_attention/multihead_attention/q/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_16/transformer/transformer/body/encoder/layer_5/self_attention/multihead_attention/k/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_16/transformer/transformer/body/encoder/layer_5/self_attention/multihead_attention/v/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_16/transformer/transformer/body/encoder/layer_5/self_attention/multihead_attention/output_transform/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_16/transformer/transformer/body/encoder/layer_5/ffn/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_16/transformer/transformer/body/encoder/layer_5/ffn/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/parallel_0_16/transformer/transformer/body/encoder/layer_5/ffn/conv1/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_16/transformer/transformer/body/encoder/layer_5/ffn/conv1/BiasAdd/ReadVariableOp',\n",
       " 'transformer/parallel_0_16/transformer/transformer/body/encoder/layer_5/ffn/conv2/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_16/transformer/transformer/body/encoder/layer_5/ffn/conv2/BiasAdd/ReadVariableOp',\n",
       " 'transformer/parallel_0_16/transformer/transformer/body/encoder/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_16/transformer/transformer/body/encoder/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/parallel_0_16/transformer/transformer/body/decoder/layer_0/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_16/transformer/transformer/body/decoder/layer_0/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/parallel_0_16/transformer/transformer/body/decoder/layer_0/self_attention/multihead_attention/q/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_16/transformer/transformer/body/decoder/layer_0/self_attention/multihead_attention/k/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_16/transformer/transformer/body/decoder/layer_0/self_attention/multihead_attention/v/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_16/transformer/transformer/body/decoder/layer_0/self_attention/multihead_attention/output_transform/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_16/transformer/transformer/body/decoder/layer_0/encdec_attention/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_16/transformer/transformer/body/decoder/layer_0/encdec_attention/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/parallel_0_16/transformer/transformer/body/decoder/layer_0/encdec_attention/multihead_attention/q/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_16/transformer/transformer/body/decoder/layer_0/encdec_attention/multihead_attention/k/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_16/transformer/transformer/body/decoder/layer_0/encdec_attention/multihead_attention/v/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_16/transformer/transformer/body/decoder/layer_0/encdec_attention/multihead_attention/output_transform/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_16/transformer/transformer/body/decoder/layer_0_1/ffn/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_16/transformer/transformer/body/decoder/layer_0_1/ffn/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/parallel_0_16/transformer/transformer/body/decoder/layer_0_1/ffn/conv1/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_16/transformer/transformer/body/decoder/layer_0_1/ffn/conv1/BiasAdd/ReadVariableOp',\n",
       " 'transformer/parallel_0_16/transformer/transformer/body/decoder/layer_0_1/ffn/conv2/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_16/transformer/transformer/body/decoder/layer_0_1/ffn/conv2/BiasAdd/ReadVariableOp',\n",
       " 'transformer/parallel_0_16/transformer/transformer/body/decoder/layer_1/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_16/transformer/transformer/body/decoder/layer_1/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/parallel_0_16/transformer/transformer/body/decoder/layer_1/self_attention/multihead_attention/q/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_16/transformer/transformer/body/decoder/layer_1/self_attention/multihead_attention/k/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_16/transformer/transformer/body/decoder/layer_1/self_attention/multihead_attention/v/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_16/transformer/transformer/body/decoder/layer_1/self_attention/multihead_attention/output_transform/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_16/transformer/transformer/body/decoder/layer_1/encdec_attention/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_16/transformer/transformer/body/decoder/layer_1/encdec_attention/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/parallel_0_16/transformer/transformer/body/decoder/layer_1/encdec_attention/multihead_attention/q/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_16/transformer/transformer/body/decoder/layer_1/encdec_attention/multihead_attention/k/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_16/transformer/transformer/body/decoder/layer_1/encdec_attention/multihead_attention/v/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_16/transformer/transformer/body/decoder/layer_1/encdec_attention/multihead_attention/output_transform/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_16/transformer/transformer/body/decoder/layer_1_1/ffn/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_16/transformer/transformer/body/decoder/layer_1_1/ffn/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/parallel_0_16/transformer/transformer/body/decoder/layer_1_1/ffn/conv1/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_16/transformer/transformer/body/decoder/layer_1_1/ffn/conv1/BiasAdd/ReadVariableOp',\n",
       " 'transformer/parallel_0_16/transformer/transformer/body/decoder/layer_1_1/ffn/conv2/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_16/transformer/transformer/body/decoder/layer_1_1/ffn/conv2/BiasAdd/ReadVariableOp',\n",
       " 'transformer/parallel_0_16/transformer/transformer/body/decoder/layer_2/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_16/transformer/transformer/body/decoder/layer_2/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/parallel_0_16/transformer/transformer/body/decoder/layer_2/self_attention/multihead_attention/q/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_16/transformer/transformer/body/decoder/layer_2/self_attention/multihead_attention/k/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_16/transformer/transformer/body/decoder/layer_2/self_attention/multihead_attention/v/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_16/transformer/transformer/body/decoder/layer_2/self_attention/multihead_attention/output_transform/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_16/transformer/transformer/body/decoder/layer_2/encdec_attention/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_16/transformer/transformer/body/decoder/layer_2/encdec_attention/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/parallel_0_16/transformer/transformer/body/decoder/layer_2/encdec_attention/multihead_attention/q/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_16/transformer/transformer/body/decoder/layer_2/encdec_attention/multihead_attention/k/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_16/transformer/transformer/body/decoder/layer_2/encdec_attention/multihead_attention/v/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_16/transformer/transformer/body/decoder/layer_2/encdec_attention/multihead_attention/output_transform/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_16/transformer/transformer/body/decoder/layer_2_1/ffn/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_16/transformer/transformer/body/decoder/layer_2_1/ffn/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/parallel_0_16/transformer/transformer/body/decoder/layer_2_1/ffn/conv1/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_16/transformer/transformer/body/decoder/layer_2_1/ffn/conv1/BiasAdd/ReadVariableOp',\n",
       " 'transformer/parallel_0_16/transformer/transformer/body/decoder/layer_2_1/ffn/conv2/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_16/transformer/transformer/body/decoder/layer_2_1/ffn/conv2/BiasAdd/ReadVariableOp',\n",
       " 'transformer/parallel_0_16/transformer/transformer/body/decoder/layer_3/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_16/transformer/transformer/body/decoder/layer_3/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/parallel_0_16/transformer/transformer/body/decoder/layer_3/self_attention/multihead_attention/q/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_16/transformer/transformer/body/decoder/layer_3/self_attention/multihead_attention/k/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_16/transformer/transformer/body/decoder/layer_3/self_attention/multihead_attention/v/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_16/transformer/transformer/body/decoder/layer_3/self_attention/multihead_attention/output_transform/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_16/transformer/transformer/body/decoder/layer_3/encdec_attention/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_16/transformer/transformer/body/decoder/layer_3/encdec_attention/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/parallel_0_16/transformer/transformer/body/decoder/layer_3/encdec_attention/multihead_attention/q/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_16/transformer/transformer/body/decoder/layer_3/encdec_attention/multihead_attention/k/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_16/transformer/transformer/body/decoder/layer_3/encdec_attention/multihead_attention/v/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_16/transformer/transformer/body/decoder/layer_3/encdec_attention/multihead_attention/output_transform/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_16/transformer/transformer/body/decoder/layer_3_1/ffn/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_16/transformer/transformer/body/decoder/layer_3_1/ffn/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/parallel_0_16/transformer/transformer/body/decoder/layer_3_1/ffn/conv1/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_16/transformer/transformer/body/decoder/layer_3_1/ffn/conv1/BiasAdd/ReadVariableOp',\n",
       " 'transformer/parallel_0_16/transformer/transformer/body/decoder/layer_3_1/ffn/conv2/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_16/transformer/transformer/body/decoder/layer_3_1/ffn/conv2/BiasAdd/ReadVariableOp',\n",
       " 'transformer/parallel_0_16/transformer/transformer/body/decoder/layer_4/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_16/transformer/transformer/body/decoder/layer_4/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/parallel_0_16/transformer/transformer/body/decoder/layer_4/self_attention/multihead_attention/q/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_16/transformer/transformer/body/decoder/layer_4/self_attention/multihead_attention/k/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_16/transformer/transformer/body/decoder/layer_4/self_attention/multihead_attention/v/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_16/transformer/transformer/body/decoder/layer_4/self_attention/multihead_attention/output_transform/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_16/transformer/transformer/body/decoder/layer_4/encdec_attention/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_16/transformer/transformer/body/decoder/layer_4/encdec_attention/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/parallel_0_16/transformer/transformer/body/decoder/layer_4/encdec_attention/multihead_attention/q/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_16/transformer/transformer/body/decoder/layer_4/encdec_attention/multihead_attention/k/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_16/transformer/transformer/body/decoder/layer_4/encdec_attention/multihead_attention/v/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_16/transformer/transformer/body/decoder/layer_4/encdec_attention/multihead_attention/output_transform/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_16/transformer/transformer/body/decoder/layer_4_1/ffn/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_16/transformer/transformer/body/decoder/layer_4_1/ffn/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/parallel_0_16/transformer/transformer/body/decoder/layer_4_1/ffn/conv1/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_16/transformer/transformer/body/decoder/layer_4_1/ffn/conv1/BiasAdd/ReadVariableOp',\n",
       " 'transformer/parallel_0_16/transformer/transformer/body/decoder/layer_4_1/ffn/conv2/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_16/transformer/transformer/body/decoder/layer_4_1/ffn/conv2/BiasAdd/ReadVariableOp',\n",
       " 'transformer/parallel_0_16/transformer/transformer/body/decoder/layer_5/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_16/transformer/transformer/body/decoder/layer_5/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/parallel_0_16/transformer/transformer/body/decoder/layer_5/self_attention/multihead_attention/q/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_16/transformer/transformer/body/decoder/layer_5/self_attention/multihead_attention/k/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_16/transformer/transformer/body/decoder/layer_5/self_attention/multihead_attention/v/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_16/transformer/transformer/body/decoder/layer_5/self_attention/multihead_attention/output_transform/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_16/transformer/transformer/body/decoder/layer_5/encdec_attention/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_16/transformer/transformer/body/decoder/layer_5/encdec_attention/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/parallel_0_16/transformer/transformer/body/decoder/layer_5/encdec_attention/multihead_attention/q/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_16/transformer/transformer/body/decoder/layer_5/encdec_attention/multihead_attention/k/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_16/transformer/transformer/body/decoder/layer_5/encdec_attention/multihead_attention/v/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_16/transformer/transformer/body/decoder/layer_5/encdec_attention/multihead_attention/output_transform/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_16/transformer/transformer/body/decoder/layer_5_1/ffn/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_16/transformer/transformer/body/decoder/layer_5_1/ffn/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/parallel_0_16/transformer/transformer/body/decoder/layer_5_1/ffn/conv1/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_16/transformer/transformer/body/decoder/layer_5_1/ffn/conv1/BiasAdd/ReadVariableOp',\n",
       " 'transformer/parallel_0_16/transformer/transformer/body/decoder/layer_5_1/ffn/conv2/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_16/transformer/transformer/body/decoder/layer_5_1/ffn/conv2/BiasAdd/ReadVariableOp',\n",
       " 'transformer/parallel_0_16/transformer/transformer/body/decoder/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_16/transformer/transformer/body/decoder/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'greedy',\n",
       " 'beam',\n",
       " 'nucleus']"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "strings = ','.join(\n",
    "    [\n",
    "        n.name\n",
    "        for n in tf.get_default_graph().as_graph_def().node\n",
    "        if ('Variable' in n.op\n",
    "        or 'Placeholder' in n.name\n",
    "        or 'greedy' in n.name\n",
    "        or 'beam' in n.name\n",
    "        or 'nucleus' in n.name\n",
    "        or 'alphas' in n.name\n",
    "        or 'self/Softmax' in n.name)\n",
    "        and 'adam' not in n.name\n",
    "        and 'beta' not in n.name\n",
    "        and 'global_step' not in n.name\n",
    "        and 'modality' not in n.name\n",
    "        and 'Assign' not in n.name\n",
    "    ]\n",
    ")\n",
    "strings.split(',')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "def freeze_graph(model_dir, output_node_names):\n",
    "\n",
    "    if not tf.gfile.Exists(model_dir):\n",
    "        raise AssertionError(\n",
    "            \"Export directory doesn't exists. Please specify an export \"\n",
    "            'directory: %s' % model_dir\n",
    "        )\n",
    "\n",
    "    checkpoint = tf.train.get_checkpoint_state(model_dir)\n",
    "    input_checkpoint = checkpoint.model_checkpoint_path\n",
    "\n",
    "    absolute_model_dir = '/'.join(input_checkpoint.split('/')[:-1])\n",
    "    output_graph = absolute_model_dir + '/frozen_model.pb'\n",
    "    clear_devices = True\n",
    "    with tf.Session(graph = tf.Graph()) as sess:\n",
    "        saver = tf.train.import_meta_graph(\n",
    "            input_checkpoint + '.meta', clear_devices = clear_devices\n",
    "        )\n",
    "        saver.restore(sess, input_checkpoint)\n",
    "        output_graph_def = tf.graph_util.convert_variables_to_constants(\n",
    "            sess,\n",
    "            tf.get_default_graph().as_graph_def(),\n",
    "            output_node_names.split(','),\n",
    "        )\n",
    "        with tf.gfile.GFile(output_graph, 'wb') as f:\n",
    "            f.write(output_graph_def.SerializeToString())\n",
    "        print('%d ops in the final graph.' % len(output_graph_def.node))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Restoring parameters from transformer-small/model.ckpt\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Restoring parameters from transformer-small/model.ckpt\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From <ipython-input-27-9a7215a4e58a>:23: convert_variables_to_constants (from tensorflow.python.framework.graph_util_impl) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use `tf.compat.v1.graph_util.convert_variables_to_constants`\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From <ipython-input-27-9a7215a4e58a>:23: convert_variables_to_constants (from tensorflow.python.framework.graph_util_impl) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use `tf.compat.v1.graph_util.convert_variables_to_constants`\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/tensorflow_core/python/framework/graph_util_impl.py:277: extract_sub_graph (from tensorflow.python.framework.graph_util_impl) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use `tf.compat.v1.graph_util.extract_sub_graph`\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/tensorflow_core/python/framework/graph_util_impl.py:277: extract_sub_graph (from tensorflow.python.framework.graph_util_impl) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use `tf.compat.v1.graph_util.extract_sub_graph`\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Froze 188 variables.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Froze 188 variables.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Converted 188 variables to const ops.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Converted 188 variables to const ops.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "15839 ops in the final graph.\n"
     ]
    }
   ],
   "source": [
    "freeze_graph('transformer-small', strings)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_graph(frozen_graph_filename):\n",
    "    with tf.gfile.GFile(frozen_graph_filename, 'rb') as f:\n",
    "        graph_def = tf.GraphDef()\n",
    "        graph_def.ParseFromString(f.read())\n",
    "    with tf.Graph().as_default() as graph:\n",
    "        tf.import_graph_def(graph_def)\n",
    "    return graph"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "g = load_graph('transformer-small/frozen_model.pb')\n",
    "x = g.get_tensor_by_name('import/Placeholder:0')\n",
    "greedy = g.get_tensor_by_name('import/greedy:0')\n",
    "beam = g.get_tensor_by_name('import/beam:0')\n",
    "nucleus = g.get_tensor_by_name('import/nucleus:0')\n",
    "top_p = g.get_tensor_by_name('import/Placeholder_2:0')\n",
    "test_sess = tf.InteractiveSession(graph = g)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "g, b, n = test_sess.run([greedy, beam, nucleus], feed_dict = {x:[encoded],\n",
    "                                                             top_p: 0.7})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "('Kejayaan sukan lebih baik',\n",
       " 'Kejayaan sukan lebih baik',\n",
       " 'Kejayaan sukan besar bukan jaminan Olimpik merit - KBS')"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(encoder.decode(f[0].tolist()), \n",
    "encoder.decode(b[0].tolist()), \n",
    "encoder.decode(n[0].tolist()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
